Laan Tungir
694219d003
Add metadata_scan.c/.h: pure-C, dependency-free byte scanners for JPEG (APP1 Exif/XMP, APP13 IPTC), PNG (eXIf, denylisted tEXt/iTXt/zTXt keys), WebP (EXIF/XMP chunks), GIF (comment/XMP extensions), PDF (/Author, /Producer, /CreationDate, etc.), TIFF (ExifIFD/GPS IFD pointers), HEIC (Exif item payload). Conservative — when in doubt, flags. Wire into handle_upload_request_with_validation() after SHA-256 compute, before fopen: rejects with 415 + X-Reason: exif_detected, preserving BUD-01 content addressing (server never rewrites). Add metadata_scan_enabled config toggle (default true). Add to Makefile + Dockerfile.alpine-musl. Tests: metadata_scan_test.c unit test (20 cases) + metadata_scan_test.sh integration test (7 cases, run against live server). See ~/lt/metadata_stripping/plans/blob-metadata-stripping.md (Part 2).
234 lines
8.8 KiB
Bash
Executable File
234 lines
8.8 KiB
Bash
Executable File
#!/bin/bash
|
|
#
|
|
# metadata_scan_test.sh — Integration test for the server-side metadata
|
|
# detect-and-reject backstop.
|
|
#
|
|
# Crafts dirty (EXIF/XMP/PDF-author) and clean fixtures with exiftool, uploads
|
|
# each to a running ginxsom instance, and asserts:
|
|
# - dirty fixtures → HTTP 415 + X-Reason: exif_detected
|
|
# - clean fixtures → HTTP 200 (or 409 if already exists)
|
|
#
|
|
# Requires: curl, nak, exiftool, sha256sum, base64, jq
|
|
#
|
|
# Usage:
|
|
# ./tests/metadata_scan_test.sh # default server
|
|
# ./tests/metadata_scan_test.sh https://blossom.example.net
|
|
# SERVER_URL=... TEST_PRIVKEY=... ./tests/metadata_scan_test.sh
|
|
#
|
|
# See plans/blob-metadata-stripping.md (Part 2 / Phase 1).
|
|
|
|
set -euo pipefail
|
|
|
|
SERVER_URL="${1:-${SERVER_URL:-https://blossom.laantungir.net}}"
|
|
UPLOAD_ENDPOINT="${SERVER_URL}/upload"
|
|
# Test privkey (from the existing upload_html_test.sh). Not a production key.
|
|
TEST_PRIVKEY="${TEST_PRIVKEY:-22cc83aa57928a2800234c939240c9a6f0f44a33ea3838a860ed38930b195afd}"
|
|
|
|
RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; BLUE='\033[0;34m'; NC='\033[0m'
|
|
log_info() { echo -e "${BLUE}[INFO]${NC} $1"; }
|
|
log_pass() { echo -e "${GREEN}[PASS]${NC} $1"; }
|
|
log_fail() { echo -e "${RED}[FAIL]${NC} $1"; }
|
|
log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
|
|
|
|
PASS=0; FAIL=0
|
|
|
|
# ─── dependency check ────────────────────────────────────────────────────
|
|
for tool in curl nak exiftool sha256sum base64 jq; do
|
|
if ! command -v "$tool" >/dev/null 2>&1; then
|
|
log_fail "Missing required tool: $tool"
|
|
case "$tool" in
|
|
nak) echo " Install: https://github.com/fiatjaf/nak" ;;
|
|
exiftool) echo " Install: sudo apt install libimage-exiftool-perl" ;;
|
|
esac
|
|
exit 1
|
|
fi
|
|
done
|
|
|
|
# ─── helpers ─────────────────────────────────────────────────────────────
|
|
TMPDIR=$(mktemp -d)
|
|
trap 'rm -rf "$TMPDIR"' EXIT
|
|
|
|
# Generate a kind 24242 upload auth event and base64-encode it.
|
|
make_auth() {
|
|
local sha="$1" size="$2" ct="$3" name="$4"
|
|
local exp; exp=$(( $(date +%s) + 3600 ))
|
|
local evt
|
|
evt=$(nak event -k 24242 -c "Upload ${name}" \
|
|
--sec "$TEST_PRIVKEY" \
|
|
-t "t=upload" \
|
|
-t "x=${sha}" \
|
|
-t "size=${size}" \
|
|
-t "expiration=${exp}" 2>/dev/null)
|
|
if [[ -z "$evt" ]]; then
|
|
log_fail "nak event failed"
|
|
exit 1
|
|
fi
|
|
printf '%s' "$evt" | base64 -w 0
|
|
}
|
|
|
|
# Upload a file and print: "HTTP_STATUS X-Reason-value"
|
|
upload_file() {
|
|
local file="$1" ct="$2" name="$3"
|
|
local sha size auth resp status xreason
|
|
sha=$(sha256sum "$file" | cut -d' ' -f1)
|
|
size=$(stat -c%s "$file")
|
|
auth=$(make_auth "$sha" "$size" "$ct" "$name")
|
|
resp=$(mktemp)
|
|
status=$(curl -s -w "%{http_code}" -X PUT \
|
|
-H "Authorization: Nostr ${auth}" \
|
|
-H "Content-Type: ${ct}" \
|
|
--data-binary "@${file}" \
|
|
-D "$TMPDIR/headers.txt" \
|
|
"${UPLOAD_ENDPOINT}" -o "$resp" 2>/dev/null || echo "000")
|
|
xreason=$(grep -i '^X-Reason:' "$TMPDIR/headers.txt" 2>/dev/null | head -1 | sed -E 's/^[Xx]-[Rr]eason:[[:space:]]*//I' | tr -d '\r' || true)
|
|
rm -f "$resp"
|
|
printf '%s\t%s' "$status" "$xreason"
|
|
}
|
|
|
|
# Assert an upload result. $1=expected_status, $2=actual, $3=test_name, $4=reason_substring (optional)
|
|
assert_status() {
|
|
local expected="$1" actual="$2" name="$3" reason_sub="${4:-}"
|
|
local status_part; status_part=$(printf '%s' "$actual" | cut -f1)
|
|
local reason_part; reason_part=$(printf '%s' "$actual" | cut -f2)
|
|
if [[ "$status_part" == "$expected" ]]; then
|
|
if [[ -z "$reason_sub" ]] || [[ "$reason_part" == *"$reason_sub"* ]]; then
|
|
log_pass "$name (HTTP $status_part${reason_part:+, X-Reason: $reason_part})"
|
|
PASS=$((PASS+1))
|
|
else
|
|
log_fail "$name: status OK ($status_part) but X-Reason missing '$reason_sub' (got: '$reason_part')"
|
|
FAIL=$((FAIL+1))
|
|
fi
|
|
else
|
|
log_fail "$name: expected HTTP $expected, got $status_part (X-Reason: $reason_part)"
|
|
FAIL=$((FAIL+1))
|
|
fi
|
|
}
|
|
|
|
# ─── craft fixtures ──────────────────────────────────────────────────────
|
|
log_info "Crafting test fixtures in $TMPDIR"
|
|
|
|
# 1. Clean JPEG (no EXIF) — 1x1 pixel via ImageMagick if available, else raw bytes.
|
|
CLEAN_JPEG="$TMPDIR/clean.jpg"
|
|
if command -v convert >/dev/null 2>&1; then
|
|
convert -size 1x1 xc:white "$CLEAN_JPEG"
|
|
else
|
|
# Minimal valid JPEG: FF D8 FF E0 00 10 JFIF... FF D9
|
|
printf '\xff\xd8\xff\xe0\x00\x10JFIF\x00\x01\x01\x00\x00\x01\x00\x01\x00\x00\xff\xd9' > "$CLEAN_JPEG"
|
|
fi
|
|
|
|
# 2. Dirty JPEG with EXIF GPS + Author (via exiftool).
|
|
DIRTY_JPEG="$TMPDIR/dirty_exif.jpg"
|
|
cp "$CLEAN_JPEG" "$DIRTY_JPEG"
|
|
exiftool -overwrite_original -q \
|
|
-GPSLatitude=48.8584 -GPSLongitude=2.2945 -GPSLatitudeRef=N -GPSLongitudeRef=E \
|
|
-Author="Test Author" -Make="TestCam" -Model="TestModel" \
|
|
"$DIRTY_JPEG" 2>/dev/null || true
|
|
|
|
# 3. Dirty JPEG with XMP packet.
|
|
DIRTY_XMP_JPEG="$TMPDIR/dirty_xmp.jpg"
|
|
cp "$CLEAN_JPEG" "$DIRTY_XMP_JPEG"
|
|
exiftool -overwrite_original -q \
|
|
-XMP-dc:Creator="XMP Test Creator" -XMP-dc:Rights="CC BY" \
|
|
"$DIRTY_XMP_JPEG" 2>/dev/null || true
|
|
|
|
# 4. Clean PNG (no text chunks).
|
|
CLEAN_PNG="$TMPDIR/clean.png"
|
|
if command -v convert >/dev/null 2>&1; then
|
|
convert -size 1x1 xc:white "$CLEAN_PNG"
|
|
else
|
|
# Minimal 1x1 PNG (IHDR + IDAT + IEND)
|
|
printf '\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01\x08\x02\x00\x00\x00\x90wS\xde\x00\x00\x00\x0cIDATx\x9cc\xf8\xcf\xc0\x00\x00\x00\x03\x00\x01\x8a\xeb\x18\x95\x00\x00\x00\x00IEND\xaeB`\x82' > "$CLEAN_PNG"
|
|
fi
|
|
|
|
# 5. Dirty PNG with eXIf chunk.
|
|
DIRTY_PNG="$TMPDIR/dirty_exif.png"
|
|
cp "$CLEAN_PNG" "$DIRTY_PNG"
|
|
exiftool -overwrite_original -q \
|
|
-Author="PNG Author" -Comment="secret" \
|
|
"$DIRTY_PNG" 2>/dev/null || true
|
|
|
|
# 6. Dirty PDF with /Author.
|
|
DIRTY_PDF="$TMPDIR/dirty.pdf"
|
|
cat > "$DIRTY_PDF" <<'PDF'
|
|
%PDF-1.4
|
|
1 0 obj
|
|
<< /Type /Catalog /Pages 2 0 R >>
|
|
endobj
|
|
2 0 obj
|
|
<< /Type /Pages /Kids [] /Count 0 >>
|
|
endobj
|
|
<< /Author (Secret Author) /Producer (Test Producer) /CreationDate (D:20260101000000) >>
|
|
trailer
|
|
<< /Root 1 0 R >>
|
|
%%EOF
|
|
PDF
|
|
|
|
# 7. Clean PDF (no author metadata).
|
|
CLEAN_PDF="$TMPDIR/clean.pdf"
|
|
cat > "$CLEAN_PDF" <<'PDF'
|
|
%PDF-1.4
|
|
1 0 obj
|
|
<< /Type /Catalog /Pages 2 0 R >>
|
|
endobj
|
|
2 0 obj
|
|
<< /Type /Pages /Kids [] /Count 0 >>
|
|
endobj
|
|
trailer
|
|
<< /Root 1 0 R >>
|
|
%%EOF
|
|
PDF
|
|
|
|
# ─── run tests ───────────────────────────────────────────────────────────
|
|
echo ""
|
|
log_info "Server: $SERVER_URL"
|
|
log_info "Upload endpoint: $UPLOAD_ENDPOINT"
|
|
echo ""
|
|
|
|
log_info "Test 1: clean JPEG (should pass)"
|
|
r=$(upload_file "$CLEAN_JPEG" "image/jpeg" "clean.jpg")
|
|
assert_status "200" "$r" "clean JPEG accepted" || true
|
|
# 409 is also acceptable (blob already exists from a prior run)
|
|
if [[ "$(printf '%s' "$r" | cut -f1)" == "409" ]]; then
|
|
log_warn " (409 — blob already exists from prior run; treating as pass)"
|
|
PASS=$((PASS+1)); FAIL=$((FAIL-1))
|
|
fi
|
|
|
|
log_info "Test 2: JPEG with EXIF GPS + Author (should be rejected)"
|
|
r=$(upload_file "$DIRTY_JPEG" "image/jpeg" "dirty_exif.jpg")
|
|
assert_status "415" "$r" "EXIF JPEG rejected" "exif_detected" || true
|
|
|
|
log_info "Test 3: JPEG with XMP packet (should be rejected)"
|
|
r=$(upload_file "$DIRTY_XMP_JPEG" "image/jpeg" "dirty_xmp.jpg")
|
|
assert_status "415" "$r" "XMP JPEG rejected" "exif_detected" || true
|
|
|
|
log_info "Test 4: clean PNG (should pass)"
|
|
r=$(upload_file "$CLEAN_PNG" "image/png" "clean.png")
|
|
assert_status "200" "$r" "clean PNG accepted" || true
|
|
if [[ "$(printf '%s' "$r" | cut -f1)" == "409" ]]; then
|
|
log_warn " (409 — blob already exists; treating as pass)"
|
|
PASS=$((PASS+1)); FAIL=$((FAIL-1))
|
|
fi
|
|
|
|
log_info "Test 5: PNG with EXIF/text chunks (should be rejected)"
|
|
r=$(upload_file "$DIRTY_PNG" "image/png" "dirty_exif.png")
|
|
assert_status "415" "$r" "dirty PNG rejected" "exif_detected" || true
|
|
|
|
log_info "Test 6: PDF with /Author (should be rejected)"
|
|
r=$(upload_file "$DIRTY_PDF" "application/pdf" "dirty.pdf")
|
|
assert_status "415" "$r" "dirty PDF rejected" "exif_detected" || true
|
|
|
|
log_info "Test 7: clean PDF (should pass)"
|
|
r=$(upload_file "$CLEAN_PDF" "application/pdf" "clean.pdf")
|
|
assert_status "200" "$r" "clean PDF accepted" || true
|
|
if [[ "$(printf '%s' "$r" | cut -f1)" == "409" ]]; then
|
|
log_warn " (409 — blob already exists; treating as pass)"
|
|
PASS=$((PASS+1)); FAIL=$((FAIL-1))
|
|
fi
|
|
|
|
# ─── summary ─────────────────────────────────────────────────────────────
|
|
echo ""
|
|
echo "=========================================="
|
|
echo -e " ${GREEN}PASSED: $PASS${NC} ${RED}FAILED: $FAIL${NC}"
|
|
echo "=========================================="
|
|
[[ "$FAIL" -eq 0 ]] && exit 0 || exit 1
|