Laan Tungir
694219d003
Add metadata_scan.c/.h: pure-C, dependency-free byte scanners for JPEG (APP1 Exif/XMP, APP13 IPTC), PNG (eXIf, denylisted tEXt/iTXt/zTXt keys), WebP (EXIF/XMP chunks), GIF (comment/XMP extensions), PDF (/Author, /Producer, /CreationDate, etc.), TIFF (ExifIFD/GPS IFD pointers), HEIC (Exif item payload). Conservative — when in doubt, flags. Wire into handle_upload_request_with_validation() after SHA-256 compute, before fopen: rejects with 415 + X-Reason: exif_detected, preserving BUD-01 content addressing (server never rewrites). Add metadata_scan_enabled config toggle (default true). Add to Makefile + Dockerfile.alpine-musl. Tests: metadata_scan_test.c unit test (20 cases) + metadata_scan_test.sh integration test (7 cases, run against live server). See ~/lt/metadata_stripping/plans/blob-metadata-stripping.md (Part 2).
437 lines
19 KiB
C
437 lines
19 KiB
C
/*
|
|
* metadata_scan.c — Privacy-metadata detection for uploaded blobs.
|
|
*
|
|
* Pure-C, dependency-free byte scanners for JPEG / PNG / WebP / GIF / PDF /
|
|
* TIFF / HEIC. Conservative: when in doubt, flag. See metadata_scan.h and
|
|
* plans/blob-metadata-stripping.md (Part 2) for the design.
|
|
*
|
|
* The scanner never rewrites bytes — it only detects, so the caller can
|
|
* reject with 415 + X-Reason and let the client retry with stripped bytes.
|
|
* This preserves BUD-01 content addressing (the client-signed hash stays
|
|
* valid for the bytes the server stores).
|
|
*/
|
|
|
|
#include "metadata_scan.h"
|
|
#include "app_log.h"
|
|
#include "ginxsom.h"
|
|
#include <string.h>
|
|
#include <stdio.h>
|
|
|
|
/* ─── config toggle ────────────────────────────────────────────────────── */
|
|
/* Read metadata_scan_enabled from the config table. Default true (server is
|
|
* the backstop). Mirrors nip94_enabled() in bud08.c. */
|
|
|
|
int metadata_scan_enabled(void) {
|
|
sqlite3 *db;
|
|
sqlite3_stmt *stmt;
|
|
int rc, enabled = 1; /* default enabled */
|
|
|
|
rc = sqlite3_open_v2(g_db_path, &db, SQLITE_OPEN_READONLY, NULL);
|
|
if (rc) {
|
|
return 1; /* default enabled on DB error */
|
|
}
|
|
|
|
const char *sql = "SELECT value FROM config WHERE key = 'metadata_scan_enabled'";
|
|
rc = sqlite3_prepare_v2(db, sql, -1, &stmt, NULL);
|
|
if (rc == SQLITE_OK) {
|
|
rc = sqlite3_step(stmt);
|
|
if (rc == SQLITE_ROW) {
|
|
const char *value = (const char *)sqlite3_column_text(stmt, 0);
|
|
enabled = (value && strcmp(value, "true") == 0) ? 1 : 0;
|
|
}
|
|
sqlite3_finalize(stmt);
|
|
}
|
|
sqlite3_close(db);
|
|
return enabled;
|
|
}
|
|
|
|
/* ─── helpers ──────────────────────────────────────────────────────────── */
|
|
|
|
/* Case-insensitive prefix match. */
|
|
static int starts_with_ci(const unsigned char *hay, size_t hay_len,
|
|
const char *needle) {
|
|
size_t n = strlen(needle);
|
|
if (hay_len < n) return 0;
|
|
for (size_t i = 0; i < n; i++) {
|
|
char h = (char)hay[i], nd = needle[i];
|
|
if (h >= 'A' && h <= 'Z') h += 32;
|
|
if (nd >= 'A' && nd <= 'Z') nd += 32;
|
|
if (h != nd) return 0;
|
|
}
|
|
return 1;
|
|
}
|
|
|
|
/* Read a big-endian 16-bit value. */
|
|
static unsigned int rd_be16(const unsigned char *p) {
|
|
return ((unsigned int)p[0] << 8) | (unsigned int)p[1];
|
|
}
|
|
|
|
/* Read a big-endian 32-bit value. */
|
|
static unsigned int rd_be32(const unsigned char *p) {
|
|
return ((unsigned int)p[0] << 24) | ((unsigned int)p[1] << 16) |
|
|
((unsigned int)p[2] << 8) | (unsigned int)p[3];
|
|
}
|
|
|
|
/* Read a little-endian 32-bit value. */
|
|
static unsigned int rd_le32(const unsigned char *p) {
|
|
return ((unsigned int)p[3] << 24) | ((unsigned int)p[2] << 16) |
|
|
((unsigned int)p[1] << 8) | (unsigned int)p[0];
|
|
}
|
|
|
|
/* Write a short reason into the caller's buffer. */
|
|
static void set_reason(char *out, size_t out_size, const char *reason) {
|
|
if (!out || out_size == 0) return;
|
|
snprintf(out, out_size, "exif_detected: %s", reason);
|
|
}
|
|
|
|
/* A found-forbidden shortcut macro: set reason and return. */
|
|
#define FORBIDDEN(reason_str) \
|
|
do { \
|
|
set_reason(reason_out, reason_out_size, reason_str); \
|
|
app_log(LOG_INFO, "METADATA_SCAN: rejected — %s\n", reason_str); \
|
|
return META_SCAN_FORBIDDEN_FOUND; \
|
|
} while (0)
|
|
|
|
/* ─── JPEG ─────────────────────────────────────────────────────────────── */
|
|
/* Segments: FF D8 FF Ex LL LL <payload>. We walk markers from offset 2. */
|
|
|
|
static meta_scan_result_t scan_jpeg(const unsigned char *data, size_t size,
|
|
char *reason_out, size_t reason_out_size) {
|
|
size_t i = 2; /* skip FF D8 */
|
|
while (i + 4 <= size) {
|
|
if (data[i] != 0xFF) break; /* not a marker — image data */
|
|
unsigned char marker = data[i + 1];
|
|
/* SOI (D8), EOI (D9), RSTn (D0-D7) have no length payload. */
|
|
if (marker == 0xD8 || marker == 0xD9) { i += 2; continue; }
|
|
if (marker >= 0xD0 && marker <= 0xD7) { i += 2; continue; }
|
|
/* SOS (DA) — start of scan; image data follows, stop walking. */
|
|
if (marker == 0xDA) break;
|
|
if (i + 4 > size) break;
|
|
unsigned int seg_len = rd_be16(data + i + 2);
|
|
if (seg_len < 2 || i + 2 + seg_len > size) break;
|
|
const unsigned char *payload = data + i + 4; /* after FF Ex LL LL */
|
|
size_t payload_len = seg_len - 2;
|
|
|
|
/* APP1 — EXIF or XMP. */
|
|
if (marker == 0xE1) {
|
|
if (payload_len >= 6 && memcmp(payload, "Exif\0\0", 6) == 0) {
|
|
FORBIDDEN("JPEG APP1 Exif segment");
|
|
}
|
|
if (payload_len >= 29 &&
|
|
starts_with_ci(payload, payload_len,
|
|
"http://ns.adobe.com/xap/1.0/")) {
|
|
FORBIDDEN("JPEG APP1 XMP segment");
|
|
}
|
|
}
|
|
/* APP13 — Photoshop / IPTC (8BIM). */
|
|
if (marker == 0xED) {
|
|
if (payload_len >= 4 && memcmp(payload, "8BIM", 4) == 0) {
|
|
FORBIDDEN("JPEG APP13 Photoshop/IPTC 8BIM segment");
|
|
}
|
|
}
|
|
/* APP2 ICC_PROFILE is allowed (color, not PII). */
|
|
i += 2 + seg_len;
|
|
}
|
|
return META_SCAN_OK;
|
|
}
|
|
|
|
/* ─── PNG ──────────────────────────────────────────────────────────────── */
|
|
/* Chunks: LL LL LL LL type <data> CRC. We walk from offset 8 (after sig). */
|
|
|
|
/* Denylisted tEXt/iTXt/zTXt keys (case-insensitive prefix). */
|
|
static int png_text_key_denied(const unsigned char *key, size_t key_len) {
|
|
static const char *denied[] = {
|
|
"Software", "Comment", "Author", "Description", "Copyright",
|
|
"XML:com.adobe.xmp", "Raw profile type exif", "Raw profile type",
|
|
"Source", "Creation Time", "Title", "Disclaimer", "Warning",
|
|
"Label", NULL
|
|
};
|
|
for (int k = 0; denied[k]; k++) {
|
|
size_t n = strlen(denied[k]);
|
|
if (key_len >= n) {
|
|
int match = 1;
|
|
for (size_t i = 0; i < n; i++) {
|
|
char a = (char)key[i], b = denied[k][i];
|
|
if (a >= 'A' && a <= 'Z') a += 32;
|
|
if (b >= 'A' && b <= 'Z') b += 32;
|
|
if (a != b) { match = 0; break; }
|
|
}
|
|
if (match) return 1;
|
|
}
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
static meta_scan_result_t scan_png(const unsigned char *data, size_t size,
|
|
char *reason_out, size_t reason_out_size) {
|
|
size_t i = 8; /* skip 8-byte signature */
|
|
while (i + 8 <= size) {
|
|
unsigned int chunk_len = rd_be32(data + i);
|
|
char type[5];
|
|
memcpy(type, data + i + 4, 4); type[4] = '\0';
|
|
/* chunk data starts at i+8, CRC at i+8+chunk_len */
|
|
if (i + 8 + chunk_len + 4 > size) break;
|
|
const unsigned char *cdata = data + i + 8;
|
|
|
|
if (memcmp(type, "eXIf", 4) == 0) {
|
|
FORBIDDEN("PNG eXIf chunk");
|
|
}
|
|
if (memcmp(type, "tEXt", 4) == 0 || memcmp(type, "zTXt", 4) == 0) {
|
|
/* key is null-terminated ASCII up to chunk_len */
|
|
size_t klen = 0;
|
|
while (klen < chunk_len && cdata[klen] != 0) klen++;
|
|
if (klen > 0 && png_text_key_denied(cdata, klen)) {
|
|
FORBIDDEN("PNG tEXt/zTXt chunk with denylisted key");
|
|
}
|
|
}
|
|
if (memcmp(type, "iTXt", 4) == 0) {
|
|
/* iTXt: keyword \0 compression_flag compression_method ... */
|
|
size_t klen = 0;
|
|
while (klen < chunk_len && cdata[klen] != 0) klen++;
|
|
if (klen > 0 && png_text_key_denied(cdata, klen)) {
|
|
FORBIDDEN("PNG iTXt chunk with denylisted key");
|
|
}
|
|
}
|
|
/* IEND ends the stream. */
|
|
if (memcmp(type, "IEND", 4) == 0) break;
|
|
i += 8 + chunk_len + 4;
|
|
}
|
|
return META_SCAN_OK;
|
|
}
|
|
|
|
/* ─── WebP ─────────────────────────────────────────────────────────────── */
|
|
/* RIFF....WEBP then VP8/VP8L/VP8X chunks. We scan the container for EXIF */
|
|
/* and XMP sub-chunks. */
|
|
|
|
static meta_scan_result_t scan_webp(const unsigned char *data, size_t size,
|
|
char *reason_out, size_t reason_out_size) {
|
|
/* RIFF <4 size> WEBP ... then sub-chunks: <4 type><4 size><data> */
|
|
if (size < 12) return META_SCAN_OK;
|
|
size_t i = 12;
|
|
while (i + 8 <= size) {
|
|
char type[5];
|
|
memcpy(type, data + i, 4); type[4] = '\0';
|
|
unsigned int clen = rd_le32(data + i + 4);
|
|
if (i + 8 + clen > size) break;
|
|
if (memcmp(type, "EXIF", 4) == 0) {
|
|
FORBIDDEN("WebP EXIF chunk");
|
|
}
|
|
if (memcmp(type, "XMP ", 4) == 0) {
|
|
FORBIDDEN("WebP XMP chunk");
|
|
}
|
|
/* chunks are padded to even length */
|
|
i += 8 + clen + (clen & 1);
|
|
}
|
|
return META_SCAN_OK;
|
|
}
|
|
|
|
/* ─── GIF ──────────────────────────────────────────────────────────────── */
|
|
/* 0x21 0xFE = comment extension; 0x21 0xFF = application extension. */
|
|
|
|
static meta_scan_result_t scan_gif(const unsigned char *data, size_t size,
|
|
char *reason_out, size_t reason_out_size) {
|
|
size_t i = 6; /* skip GIF87a/89a */
|
|
/* skip logical screen descriptor (7 bytes) + optional global color table */
|
|
if (i + 7 > size) return META_SCAN_OK;
|
|
unsigned char packed = data[i + 4];
|
|
i += 7;
|
|
if (packed & 0x80) {
|
|
unsigned int gct_size = 3 * (1 << ((packed & 0x07) + 1));
|
|
i += gct_size;
|
|
}
|
|
/* walk blocks */
|
|
while (i < size) {
|
|
unsigned char b = data[i];
|
|
if (b == 0x3B) break; /* trailer */
|
|
if (b == 0x21 && i + 1 < size) {
|
|
unsigned char label = data[i + 1];
|
|
if (label == 0xFE) {
|
|
FORBIDDEN("GIF comment extension (0x21 0xFE)");
|
|
}
|
|
if (label == 0xFF) {
|
|
/* application extension: check for XMP Data */
|
|
if (i + 14 < size && memcmp(data + i + 3, "XMP Data", 8) == 0) {
|
|
FORBIDDEN("GIF XMP Data application extension");
|
|
}
|
|
}
|
|
/* skip sub-blocks: 0x21 label then size-prefixed sub-blocks */
|
|
i += 2;
|
|
while (i < size && data[i] != 0) {
|
|
unsigned char sub = data[i];
|
|
i += 1 + sub;
|
|
}
|
|
i++; /* skip 0 terminator */
|
|
continue;
|
|
}
|
|
if (b == 0x2C) { /* image descriptor */
|
|
i += 10;
|
|
if (i >= size) break;
|
|
unsigned char ipacked = data[i - 1];
|
|
if (ipacked & 0x80) {
|
|
unsigned int lct = 3 * (1 << ((ipacked & 0x07) + 1));
|
|
i += lct;
|
|
}
|
|
i++; /* LZW min code size */
|
|
while (i < size && data[i] != 0) {
|
|
unsigned char sub = data[i];
|
|
i += 1 + sub;
|
|
}
|
|
i++;
|
|
continue;
|
|
}
|
|
/* unknown block — bail to avoid mis-parsing */
|
|
break;
|
|
}
|
|
return META_SCAN_OK;
|
|
}
|
|
|
|
/* ─── PDF ──────────────────────────────────────────────────────────────── */
|
|
/* Scan for forbidden dictionary keys in the raw byte stream. PDF is not */
|
|
/* cleanly parseable without a full lexer, but the keys appear as literal */
|
|
/* ASCII tokens (/Author, /Producer, etc.) and a byte scan is sufficient */
|
|
/* for detection. We avoid matching inside stream content by also requiring */
|
|
/* the token to look like a dictionary entry (preceded by whitespace/<</{). */
|
|
|
|
static int pdf_key_present(const unsigned char *data, size_t size,
|
|
const char *key) {
|
|
size_t klen = strlen(key);
|
|
if (size < klen) return 0;
|
|
for (size_t i = 0; i + klen <= size; i++) {
|
|
if (memcmp(data + i, key, klen) == 0) {
|
|
/* Check the preceding byte is a PDF delimiter to reduce false
|
|
* positives from stream pixel data that happens to contain the
|
|
* byte sequence. */
|
|
if (i == 0) return 1;
|
|
unsigned char prev = data[i - 1];
|
|
if (prev == '(' || prev == '<' || prev == '{' ||
|
|
prev == ' ' || prev == '\n' || prev == '\r' ||
|
|
prev == '\t' || prev == '/' || prev == '>' || prev == '[') {
|
|
return 1;
|
|
}
|
|
}
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
static meta_scan_result_t scan_pdf(const unsigned char *data, size_t size,
|
|
char *reason_out, size_t reason_out_size) {
|
|
static const char *keys[] = {
|
|
"/Author", "/Title", "/Subject", "/Keywords", "/Creator",
|
|
"/Producer", "/CreationDate", "/ModDate", "/XMP", "/Metadata",
|
|
NULL
|
|
};
|
|
for (int k = 0; keys[k]; k++) {
|
|
if (pdf_key_present(data, size, keys[k])) {
|
|
char buf[64];
|
|
snprintf(buf, sizeof(buf), "PDF %s entry", keys[k]);
|
|
FORBIDDEN(buf);
|
|
}
|
|
}
|
|
return META_SCAN_OK;
|
|
}
|
|
|
|
/* ─── TIFF ─────────────────────────────────────────────────────────────── */
|
|
/* TIFF is essentially all EXIF. Flag any TIFF unless it's a trivial */
|
|
/* baseline (just IFD0 with no EXIF sub-IFD). For simplicity and safety, */
|
|
/* flag all TIFFs that contain an ExifIFD pointer or GPS IFD pointer. */
|
|
|
|
static meta_scan_result_t scan_tiff(const unsigned char *data, size_t size,
|
|
char *reason_out, size_t reason_out_size) {
|
|
/* Determine endianness. */
|
|
int big_endian;
|
|
if (size >= 2 && data[0] == 'I' && data[1] == 'I') big_endian = 0;
|
|
else if (size >= 2 && data[0] == 'M' && data[1] == 'M') big_endian = 1;
|
|
else return META_SCAN_OK; /* not actually TIFF */
|
|
|
|
if (size < 8) return META_SCAN_OK;
|
|
/* magic 42 at offset 2 */
|
|
unsigned int magic = big_endian ? rd_be16(data + 2)
|
|
: (unsigned int)(data[2] | (data[3] << 8));
|
|
if (magic != 42) return META_SCAN_OK;
|
|
|
|
/* offset to first IFD */
|
|
unsigned int ifd_off = big_endian ? rd_be32(data + 4) : rd_le32(data + 4);
|
|
if (ifd_off + 2 > size) return META_SCAN_OK;
|
|
|
|
/* Walk IFD0 entries looking for ExifIFD (0x8769) or GPSIFD (0x8825). */
|
|
unsigned int entry_count = big_endian ? rd_be16(data + ifd_off)
|
|
: (unsigned int)(data[ifd_off] | (data[ifd_off + 1] << 8));
|
|
for (unsigned int e = 0; e < entry_count; e++) {
|
|
size_t off = ifd_off + 2 + e * 12;
|
|
if (off + 12 > size) break;
|
|
unsigned int tag = big_endian ? rd_be16(data + off)
|
|
: (unsigned int)(data[off] | (data[off + 1] << 8));
|
|
if (tag == 0x8769) FORBIDDEN("TIFF ExifIFD sub-directory");
|
|
if (tag == 0x8825) FORBIDDEN("TIFF GPS IFD");
|
|
}
|
|
return META_SCAN_OK;
|
|
}
|
|
|
|
/* ─── HEIC/HEIF ────────────────────────────────────────────────────────── */
|
|
/* Full HEIC box parsing is complex; a byte-pattern scan for the Exif */
|
|
/* item payload ("Exif\0\0") is sufficient for detection. */
|
|
|
|
static meta_scan_result_t scan_heic(const unsigned char *data, size_t size,
|
|
char *reason_out, size_t reason_out_size) {
|
|
/* Search for "Exif\0\0" anywhere in the file. */
|
|
static const unsigned char exif_sig[] = {'E','x','i','f',0,0};
|
|
if (size < sizeof(exif_sig)) return META_SCAN_OK;
|
|
for (size_t i = 0; i + sizeof(exif_sig) <= size; i++) {
|
|
if (memcmp(data + i, exif_sig, sizeof(exif_sig)) == 0) {
|
|
FORBIDDEN("HEIC/HEIF Exif item payload");
|
|
}
|
|
}
|
|
return META_SCAN_OK;
|
|
}
|
|
|
|
/* ─── dispatcher ───────────────────────────────────────────────────────── */
|
|
|
|
meta_scan_result_t metadata_scan(const unsigned char *data,
|
|
size_t size,
|
|
const char *content_type,
|
|
char *reason_out,
|
|
size_t reason_out_size) {
|
|
if (!data || size == 0) return META_SCAN_OK;
|
|
|
|
/* Clear reason buffer. */
|
|
if (reason_out && reason_out_size > 0) reason_out[0] = '\0';
|
|
|
|
/* Format detection by magic bytes. */
|
|
if (size >= 3 && data[0] == 0xFF && data[1] == 0xD8 && data[2] == 0xFF) {
|
|
return scan_jpeg(data, size, reason_out, reason_out_size);
|
|
}
|
|
if (size >= 8 && memcmp(data, "\x89PNG\r\n\x1a\n", 8) == 0) {
|
|
return scan_png(data, size, reason_out, reason_out_size);
|
|
}
|
|
if (size >= 12 && memcmp(data, "RIFF", 4) == 0 &&
|
|
memcmp(data + 8, "WEBP", 4) == 0) {
|
|
return scan_webp(data, size, reason_out, reason_out_size);
|
|
}
|
|
if (size >= 6 && (memcmp(data, "GIF87a", 6) == 0 ||
|
|
memcmp(data, "GIF89a", 6) == 0)) {
|
|
return scan_gif(data, size, reason_out, reason_out_size);
|
|
}
|
|
if (size >= 5 && memcmp(data, "%PDF-", 5) == 0) {
|
|
return scan_pdf(data, size, reason_out, reason_out_size);
|
|
}
|
|
if (size >= 4 && ((data[0] == 'I' && data[1] == 'I' &&
|
|
data[2] == 0x2A && data[3] == 0) ||
|
|
(data[0] == 'M' && data[1] == 'M' &&
|
|
data[2] == 0 && data[3] == 0x2A))) {
|
|
return scan_tiff(data, size, reason_out, reason_out_size);
|
|
}
|
|
/* HEIC: ftyp box with heic/heix/heif/heims brand at offset 4..12. */
|
|
if (size >= 12 && memcmp(data + 4, "ftyp", 4) == 0) {
|
|
const unsigned char *brand = data + 8;
|
|
if (memcmp(brand, "heic", 4) == 0 || memcmp(brand, "heix", 4) == 0 ||
|
|
memcmp(brand, "heif", 4) == 0 || memcmp(brand, "heims", 4) == 0 ||
|
|
memcmp(brand, "hevc", 4) == 0 || memcmp(brand, "heim", 4) == 0) {
|
|
return scan_heic(data, size, reason_out, reason_out_size);
|
|
}
|
|
}
|
|
|
|
/* Unknown format — allow (no known metadata vector). */
|
|
(void)content_type; /* currently unused; reserved for future hints */
|
|
return META_SCAN_OK;
|
|
}
|