Simplify checking whether a signature exists

appx.c

@@ -250,7 +250,6 @@ static const EVP_MD *appx_md_get(FILE_FORMAT_CTX *ctx);
 static ASN1_OBJECT *appx_spc_sip_info_get(u_char **p, int *plen, FILE_FORMAT_CTX *ctx);
 static PKCS7 *appx_pkcs7_contents_get(FILE_FORMAT_CTX *ctx, BIO *hash, const EVP_MD *md);
 static int appx_hash_length_get(FILE_FORMAT_CTX *ctx);
-static int appx_check_file(FILE_FORMAT_CTX *ctx, int detached);
 static int appx_verify_digests(FILE_FORMAT_CTX *ctx, PKCS7 *p7);
 static PKCS7 *appx_pkcs7_extract(FILE_FORMAT_CTX *ctx);
 static int appx_remove_pkcs7(FILE_FORMAT_CTX *ctx, BIO *hash, BIO *outdata);
@@ -266,7 +265,6 @@ FILE_FORMAT file_format_appx = {
     .data_blob_get = appx_spc_sip_info_get,
     .pkcs7_contents_get = appx_pkcs7_contents_get,
     .hash_length_get = appx_hash_length_get,
-    .check_file = appx_check_file,
     .verify_digests = appx_verify_digests,
     .pkcs7_extract = appx_pkcs7_extract,
     .remove_pkcs7 = appx_remove_pkcs7,
@@ -466,25 +464,6 @@ static int appx_hash_length_get(FILE_FORMAT_CTX *ctx)
     return ctx->appx_ctx->hashlen;
 }
 
-/*
- * Check if the signature exists.
- * [in] ctx: structure holds input and output data
- * [in] detached: embedded/detached PKCS#7 signature switch
- * [returns] 0 on error or 1 on success
- */
-static int appx_check_file(FILE_FORMAT_CTX *ctx, int detached)
-{
-    if (detached) {
-        printf("APPX format does not support detached PKCS#7 signature\n");
-        return 0; /* FAILED */
-    }
-    if (!zipEntryExist(ctx->appx_ctx->zip, APP_SIGNATURE_FILENAME)) {
-        printf("%s does not exist\n", APP_SIGNATURE_FILENAME);
-        return 0; /* FAILED */
-    }
-    return 1; /* OK */
-}
-
 /*
  * Calculate message digest and compare to value retrieved from PKCS#7 signedData.
  * [in] ctx: structure holds input and output data
@@ -534,6 +513,11 @@ static PKCS7 *appx_pkcs7_extract(FILE_FORMAT_CTX *ctx)
     const u_char *blob;
     size_t dataSize;
 
+    /* Check if the signature exists */
+    if (!zipEntryExist(ctx->appx_ctx->zip, APP_SIGNATURE_FILENAME)) {
+        printf("%s does not exist\n", APP_SIGNATURE_FILENAME);
+        return NULL; /* FAILED */
+    }
     dataSize = zipReadFileDataByName(&data, ctx->appx_ctx->zip, APP_SIGNATURE_FILENAME);
     if (dataSize <= 0) {
         return NULL; /* FAILED */

cab.c

@@ -45,7 +45,6 @@ static FILE_FORMAT_CTX *cab_ctx_new(GLOBAL_OPTIONS *options, BIO *hash, BIO *out
 static ASN1_OBJECT *cab_obsolete_link_get(u_char **p, int *plen, FILE_FORMAT_CTX *ctx);
 static PKCS7 *cab_pkcs7_contents_get(FILE_FORMAT_CTX *ctx, BIO *hash, const EVP_MD *md);
 static int cab_hash_length_get(FILE_FORMAT_CTX *ctx);
-static int cab_check_file(FILE_FORMAT_CTX *ctx, int detached);
 static u_char *cab_digest_calc(FILE_FORMAT_CTX *ctx, const EVP_MD *md);
 static int cab_verify_digests(FILE_FORMAT_CTX *ctx, PKCS7 *p7);
 static PKCS7 *cab_pkcs7_extract(FILE_FORMAT_CTX *ctx);
@@ -57,13 +56,13 @@ static int cab_append_pkcs7(FILE_FORMAT_CTX *ctx, BIO *outdata, PKCS7 *p7);
 static void cab_update_data_size(FILE_FORMAT_CTX *ctx, BIO *outdata, PKCS7 *p7);
 static BIO *cab_bio_free(BIO *hash, BIO *outdata);
 static void cab_ctx_cleanup(FILE_FORMAT_CTX *ctx, BIO *hash, BIO *outdata);
+static int cab_is_detaching_supported(void);
 
 FILE_FORMAT file_format_cab = {
     .ctx_new = cab_ctx_new,
     .data_blob_get = cab_obsolete_link_get,
     .pkcs7_contents_get = cab_pkcs7_contents_get,
     .hash_length_get = cab_hash_length_get,
-    .check_file = cab_check_file,
     .digest_calc = cab_digest_calc,
     .verify_digests = cab_verify_digests,
     .pkcs7_extract = cab_pkcs7_extract,
@@ -74,7 +73,8 @@ FILE_FORMAT file_format_cab = {
     .append_pkcs7 = cab_append_pkcs7,
     .update_data_size = cab_update_data_size,
     .bio_free = cab_bio_free,
-    .ctx_cleanup = cab_ctx_cleanup
+    .ctx_cleanup = cab_ctx_cleanup,
+    .is_detaching_supported = cab_is_detaching_supported
 };
 
 /* Prototypes */
@@ -83,6 +83,7 @@ static int cab_add_jp_attribute(PKCS7 *p7, int jp);
 static size_t cab_write_optional_names(BIO *outdata, char *indata, size_t len, uint16_t flags);
 static int cab_modify_header(FILE_FORMAT_CTX *ctx, BIO *hash, BIO *outdata);
 static int cab_add_header(FILE_FORMAT_CTX *ctx, BIO *hash, BIO *outdata);
+static int cab_check_file(FILE_FORMAT_CTX *ctx);
 
 /*
  * FILE_FORMAT method definitions
@@ -192,34 +193,6 @@ static int cab_hash_length_get(FILE_FORMAT_CTX *ctx)
     return EVP_MD_size(ctx->options->md);
 }
 
-/*
- * Check if the signature exists.
- * [in, out] ctx: structure holds input and output data
- * [in] detached: embedded/detached PKCS#7 signature switch
- * [returns] 0 on error or 1 on success
- */
-static int cab_check_file(FILE_FORMAT_CTX *ctx, int detached)
-{
-    if (!ctx) {
-        printf("Init error\n\n");
-        return 0; /* FAILED */
-    }
-    if (detached) {
-        printf("Checking the specified catalog file\n\n");
-        return 1; /* OK */
-    }
-    if (ctx->cab_ctx->header_size != 20) {
-        printf("No signature found\n\n");
-        return 0; /* FAILED */
-    }
-    if (ctx->cab_ctx->sigpos == 0 || ctx->cab_ctx->siglen == 0
-        || ctx->cab_ctx->sigpos > ctx->cab_ctx->fileend) {
-        printf("No signature found\n\n");
-        return 0; /* FAILED */
-    }
-    return 1; /* OK */
-}
-
 /*
  * Compute a message digest value of the signed or unsigned CAB file.
  * [in] ctx: structure holds input and output data
@@ -397,8 +370,7 @@ static PKCS7 *cab_pkcs7_extract(FILE_FORMAT_CTX *ctx)
 {
     const u_char *blob;
 
-    if (ctx->cab_ctx->sigpos == 0 || ctx->cab_ctx->siglen == 0
+    if (!cab_check_file(ctx)) {
-        || ctx->cab_ctx->sigpos > ctx->cab_ctx->fileend) {
         return NULL; /* FAILED */
     }
     blob = (u_char *)ctx->options->indata + ctx->cab_ctx->sigpos;
@@ -432,8 +404,7 @@ static int cab_remove_pkcs7(FILE_FORMAT_CTX *ctx, BIO *hash, BIO *outdata)
     /* squash the unused parameter warning */
     (void)hash;
 
-    if (ctx->cab_ctx->sigpos == 0 || ctx->cab_ctx->siglen == 0
+    if (!cab_check_file(ctx)) {
-        || ctx->cab_ctx->sigpos > ctx->cab_ctx->fileend) {
         return 1; /* FAILED, no signature */
     }
     buf = OPENSSL_malloc(SIZE_64K);
@@ -655,6 +626,11 @@ static void cab_ctx_cleanup(FILE_FORMAT_CTX *ctx, BIO *hash, BIO *outdata)
     OPENSSL_free(ctx);
 }
 
+static int cab_is_detaching_supported(void)
+{
+    return 1; /* OK */
+}
+
 /*
  * CAB helper functions
  */
@@ -972,6 +948,29 @@ static int cab_add_header(FILE_FORMAT_CTX *ctx, BIO *hash, BIO *outdata)
     return 1; /* OK */
 }
 
+/*
+ * Check if the signature exists.
+ * [in, out] ctx: structure holds input and output data
+ * [returns] 0 on error or 1 on success
+ */
+static int cab_check_file(FILE_FORMAT_CTX *ctx)
+{
+    if (!ctx) {
+        printf("Init error\n\n");
+        return 0; /* FAILED */
+    }
+    if (ctx->cab_ctx->header_size != 20) {
+        printf("No signature found\n\n");
+        return 0; /* FAILED */
+    }
+    if (ctx->cab_ctx->sigpos == 0 || ctx->cab_ctx->siglen == 0
+        || ctx->cab_ctx->sigpos > ctx->cab_ctx->fileend) {
+        printf("No signature found\n\n");
+        return 0; /* FAILED */
+    }
+    return 1; /* OK */
+}
+
 /*
 Local Variables:
    c-basic-offset: 4

cat.c

@@ -36,7 +36,6 @@ struct cat_ctx_st {
 
 /* FILE_FORMAT method prototypes */
 static FILE_FORMAT_CTX *cat_ctx_new(GLOBAL_OPTIONS *options, BIO *hash, BIO *outdata);
-static int cat_check_file(FILE_FORMAT_CTX *ctx, int detached);
 static int cat_verify_digests(FILE_FORMAT_CTX *ctx, PKCS7 *p7);
 static PKCS7 *cat_pkcs7_extract(FILE_FORMAT_CTX *ctx);
 static PKCS7 *cat_pkcs7_signature_new(FILE_FORMAT_CTX *ctx, BIO *hash);
@@ -46,7 +45,6 @@ static void cat_ctx_cleanup(FILE_FORMAT_CTX *ctx, BIO *hash, BIO *outdata);
 
 FILE_FORMAT file_format_cat = {
     .ctx_new = cat_ctx_new,
-    .check_file = cat_check_file,
     .verify_digests = cat_verify_digests,
     .pkcs7_extract = cat_pkcs7_extract,
     .pkcs7_signature_new = cat_pkcs7_signature_new,
@@ -64,6 +62,7 @@ static int cat_print_content_member_digest(ASN1_TYPE *content);
 static int cat_print_content_member_name(ASN1_TYPE *content);
 static void cat_print_base64(ASN1_OCTET_STRING *value);
 static void cat_print_utf16_as_ascii(ASN1_OCTET_STRING *value);
+static int cat_check_file(FILE_FORMAT_CTX *ctx);
 
 /*
  * FILE_FORMAT method definitions
@@ -118,35 +117,6 @@ static FILE_FORMAT_CTX *cat_ctx_new(GLOBAL_OPTIONS *options, BIO *hash, BIO *out
     return ctx;
 }
 
-static int cat_check_file(FILE_FORMAT_CTX *ctx, int detached)
-{
-    STACK_OF(PKCS7_SIGNER_INFO) *signer_info;
-    PKCS7_SIGNER_INFO *si;
-
-    if (!ctx) {
-        printf("Init error\n\n");
-        return 0; /* FAILED */
-    }
-    if (detached) {
-        printf("CAT format does not support detached PKCS#7 signature\n\n");
-        return 0; /* FAILED */
-    }
-    signer_info = PKCS7_get_signer_info(ctx->cat_ctx->p7);
-    if (!signer_info) {
-        printf("Failed catalog file\n\n");
-        return 0; /* FAILED */
-    }
-    si = sk_PKCS7_SIGNER_INFO_value(signer_info, 0);
-    if (!si) {
-        printf("No signature found\n\n");
-        return 0; /* FAILED */
-    }
-    if (ctx->options->verbose) {
-        (void)cat_list_content(ctx->cat_ctx->p7);
-    }
-    return 1; /* OK */
-}
-
 /*
  * ContentInfo value is the inner content of pkcs7-signedData.
  * An extra verification is not necessary when a content type data
@@ -167,6 +137,9 @@ static int cat_verify_digests(FILE_FORMAT_CTX *ctx, PKCS7 *p7)
  */
 static PKCS7 *cat_pkcs7_extract(FILE_FORMAT_CTX *ctx)
 {
+    if (!cat_check_file(ctx)) {
+        return NULL; /* FAILED */
+    }
     return PKCS7_dup(ctx->cat_ctx->p7);
 }
 
@@ -472,6 +445,35 @@ static void cat_print_utf16_as_ascii(ASN1_OCTET_STRING *value)
         putchar(isprint(data[i]) && !data[i+1] ? data[i] : '.');
 }
 
+/*
+ * Check if the signature exists.
+ * [in, out] ctx: structure holds input and output data
+ * [returns] 0 on error or 1 on success
+ */
+static int cat_check_file(FILE_FORMAT_CTX *ctx)
+{
+    STACK_OF(PKCS7_SIGNER_INFO) *signer_info;
+    PKCS7_SIGNER_INFO *si;
+
+    if (!ctx) {
+        printf("Init error\n\n");
+        return 0; /* FAILED */
+    }
+    signer_info = PKCS7_get_signer_info(ctx->cat_ctx->p7);
+    if (!signer_info) {
+        printf("Failed catalog file\n\n");
+        return 0; /* FAILED */
+    }
+    si = sk_PKCS7_SIGNER_INFO_value(signer_info, 0);
+    if (!si) {
+        printf("No signature found\n\n");
+        return 0; /* FAILED */
+    }
+    if (ctx->options->verbose) {
+        (void)cat_list_content(ctx->cat_ctx->p7);
+    }
+    return 1; /* OK */
+}
 /*
 Local Variables:
    c-basic-offset: 4

msi.c

@@ -193,7 +193,6 @@ static FILE_FORMAT_CTX *msi_ctx_new(GLOBAL_OPTIONS *options, BIO *hash, BIO *out
 static ASN1_OBJECT *msi_spc_sip_info_get(u_char **p, int *plen, FILE_FORMAT_CTX *ctx);
 static PKCS7 *msi_pkcs7_contents_get(FILE_FORMAT_CTX *ctx, BIO *hash, const EVP_MD *md);
 static int msi_hash_length_get(FILE_FORMAT_CTX *ctx);
-static int msi_check_file(FILE_FORMAT_CTX *ctx, int detached);
 static u_char *msi_digest_calc(FILE_FORMAT_CTX *ctx, const EVP_MD *md);
 static int msi_verify_digests(FILE_FORMAT_CTX *ctx, PKCS7 *p7);
 static PKCS7 *msi_pkcs7_extract(FILE_FORMAT_CTX *ctx);
@@ -204,13 +203,13 @@ static PKCS7 *msi_pkcs7_signature_new(FILE_FORMAT_CTX *ctx, BIO *hash);
 static int msi_append_pkcs7(FILE_FORMAT_CTX *ctx, BIO *outdata, PKCS7 *p7);
 static BIO *msi_bio_free(BIO *hash, BIO *outdata);
 static void msi_ctx_cleanup(FILE_FORMAT_CTX *ctx, BIO *hash, BIO *outdata);
+static int msi_is_detaching_supported(void);
 
 FILE_FORMAT file_format_msi = {
     .ctx_new = msi_ctx_new,
     .data_blob_get = msi_spc_sip_info_get,
     .pkcs7_contents_get = msi_pkcs7_contents_get,
     .hash_length_get = msi_hash_length_get,
-    .check_file = msi_check_file,
     .digest_calc = msi_digest_calc,
     .verify_digests = msi_verify_digests,
     .pkcs7_extract = msi_pkcs7_extract,
@@ -220,7 +219,8 @@ FILE_FORMAT file_format_msi = {
     .pkcs7_signature_new = msi_pkcs7_signature_new,
     .append_pkcs7 = msi_append_pkcs7,
     .bio_free = msi_bio_free,
-    .ctx_cleanup = msi_ctx_cleanup
+    .ctx_cleanup = msi_ctx_cleanup,
+    .is_detaching_supported = msi_is_detaching_supported
 };
 
 /* Prototypes */
@@ -242,6 +242,7 @@ static MSI_FILE *msi_file_new(char *buffer, uint32_t len);
 static int msi_dirent_new(MSI_FILE *msi, MSI_ENTRY *entry, MSI_DIRENT *parent, MSI_DIRENT **ret);
 static void msi_dirent_free(MSI_DIRENT *dirent);
 static int msi_prehash_dir(MSI_DIRENT *dirent, BIO *hash, int is_root);
+static int msi_check_file(FILE_FORMAT_CTX *ctx);
 
 /*
  * FILE_FORMAT method definitions
@@ -360,77 +361,6 @@ static PKCS7 *msi_pkcs7_contents_get(FILE_FORMAT_CTX *ctx, BIO *hash, const EVP_
     return pkcs7_set_content(content);
 }
 
-/*
- * [in] ctx: structure holds input and output data
- * [returns] the size of the message digest when passed an EVP_MD structure (the size of the hash)
- */
-static int msi_hash_length_get(FILE_FORMAT_CTX *ctx)
-{
-    return EVP_MD_size(ctx->options->md);
-}
-
-/*
- * Get DigitalSignature and MsiDigitalSignatureEx streams,
- * check if the signature exists.
- * [in, out] ctx: structure holds input and output data
- * [in] detached: embedded/detached PKCS#7 signature switch (unused)
- * [returns] 0 on error or 1 on successs
- */
-static int msi_check_file(FILE_FORMAT_CTX *ctx, int detached)
-{
-    char *indata = NULL;
-    uint32_t inlen;
-    MSI_ENTRY *ds, *dse = NULL;
-
-    /* squash the unused parameter warning */
-    (void)detached;
-
-    if (!ctx) {
-        printf("Init error\n\n");
-        return 0; /* FAILED */
-    }
-    if (detached) {
-        printf("Checking the specified catalog file\n\n");
-        return 1; /* OK */
-    }
-    ds = msi_signatures_get(ctx->msi_ctx->dirent, &dse);
-    if (!ds) {
-        printf("MSI file has no signature\n\n");
-        return 0; /* FAILED */
-    }
-    inlen = GET_UINT32_LE(ds->size);
-    if (inlen == 0 || inlen >= MAXREGSECT) {
-        printf("Corrupted DigitalSignature stream length 0x%08X\n", inlen);
-        return 0; /* FAILED */
-    }
-    indata = OPENSSL_malloc((size_t)inlen);
-    if (!msi_file_read(ctx->msi_ctx->msi, ds, 0, indata, inlen)) {
-        printf("DigitalSignature stream data error\n\n");
-        OPENSSL_free(indata);
-        return 0; /* FAILED */
-    }
-    if (!dse) {
-        printf("Warning: MsiDigitalSignatureEx stream doesn't exist\n");
-    } else {
-        ctx->msi_ctx->len_msiex = GET_UINT32_LE(dse->size);
-        if (ctx->msi_ctx->len_msiex == 0 || ctx->msi_ctx->len_msiex >= MAXREGSECT) {
-            printf("Corrupted MsiDigitalSignatureEx stream length 0x%08X\n",
-                ctx->msi_ctx->len_msiex);
-            OPENSSL_free(indata);
-            return 0; /* FAILED */
-        }
-        ctx->msi_ctx->p_msiex = OPENSSL_malloc((size_t)ctx->msi_ctx->len_msiex);
-        if (!msi_file_read(ctx->msi_ctx->msi, dse, 0, (char *)ctx->msi_ctx->p_msiex,
-                ctx->msi_ctx->len_msiex)) {
-            printf("MsiDigitalSignatureEx stream data error\n\n");
-            OPENSSL_free(indata);
-            return 0; /* FAILED */
-        }
-    }
-    OPENSSL_free(indata);
-    return 1; /* OK */
-}
-
 /*
  * Compute a simple sha1/sha256 message digest of the MSI file
  * for use with a catalog file.
@@ -566,7 +496,12 @@ static int msi_verify_digests(FILE_FORMAT_CTX *ctx, PKCS7 *p7)
 static PKCS7 *msi_pkcs7_extract(FILE_FORMAT_CTX *ctx)
 {
     PKCS7 *p7;
-    MSI_ENTRY *ds = msi_signatures_get(ctx->msi_ctx->dirent, NULL);
+    MSI_ENTRY *ds;
+
+    if (!msi_check_file(ctx)) {
+        return NULL; /* FAILED, no signature */
+    }
+    ds = msi_signatures_get(ctx->msi_ctx->dirent, NULL);
 
     if (!ds) {
         printf("MSI file has no signature\n");
@@ -591,6 +526,9 @@ static PKCS7 *msi_pkcs7_extract_to_nest(FILE_FORMAT_CTX *ctx)
     PKCS7 *p7;
     MSI_ENTRY *ds, *dse = NULL;
 
+    if (!msi_check_file(ctx)) {
+        return NULL; /* FAILED, no signature */
+    }
     ds = msi_signatures_get(ctx->msi_ctx->dirent, &dse);
     if (!ds) {
         printf("MSI file has no signature\n");
@@ -762,6 +700,11 @@ static void msi_ctx_cleanup(FILE_FORMAT_CTX *ctx, BIO *hash, BIO *outdata)
     OPENSSL_free(ctx);
 }
 
+static int msi_is_detaching_supported(void)
+{
+    return 1; /* OK */
+}
+
 /*
  * MSI helper functions
  */
@@ -2318,6 +2261,69 @@ static int msi_check_MsiDigitalSignatureEx(FILE_FORMAT_CTX *ctx, MSI_ENTRY *dse,
     return 1; /* OK */
 }
 
+/*
+ * [in] ctx: structure holds input and output data
+ * [returns] the size of the message digest when passed an EVP_MD structure (the size of the hash)
+ */
+static int msi_hash_length_get(FILE_FORMAT_CTX *ctx)
+{
+    return EVP_MD_size(ctx->options->md);
+}
+
+/*
+ * Get DigitalSignature and MsiDigitalSignatureEx streams
+ * to check if the signature exists.
+ * [in, out] ctx: structure holds input and output datafv
+ * [returns] 0 on error or 1 on successs
+ */
+static int msi_check_file(FILE_FORMAT_CTX *ctx)
+{
+    char *indata = NULL;
+    uint32_t inlen;
+    MSI_ENTRY *ds, *dse = NULL;
+
+    if (!ctx) {
+        printf("Init error\n\n");
+        return 0; /* FAILED */
+    }
+    ds = msi_signatures_get(ctx->msi_ctx->dirent, &dse);
+    if (!ds) {
+        printf("MSI file has no signature\n\n");
+        return 0; /* FAILED */
+    }
+    inlen = GET_UINT32_LE(ds->size);
+    if (inlen == 0 || inlen >= MAXREGSECT) {
+        printf("Corrupted DigitalSignature stream length 0x%08X\n", inlen);
+        return 0; /* FAILED */
+    }
+    indata = OPENSSL_malloc((size_t)inlen);
+    if (!msi_file_read(ctx->msi_ctx->msi, ds, 0, indata, inlen)) {
+        printf("DigitalSignature stream data error\n\n");
+        OPENSSL_free(indata);
+        return 0; /* FAILED */
+    }
+    if (!dse) {
+        printf("Warning: MsiDigitalSignatureEx stream doesn't exist\n");
+    } else {
+        ctx->msi_ctx->len_msiex = GET_UINT32_LE(dse->size);
+        if (ctx->msi_ctx->len_msiex == 0 || ctx->msi_ctx->len_msiex >= MAXREGSECT) {
+            printf("Corrupted MsiDigitalSignatureEx stream length 0x%08X\n",
+                ctx->msi_ctx->len_msiex);
+            OPENSSL_free(indata);
+            return 0; /* FAILED */
+        }
+        ctx->msi_ctx->p_msiex = OPENSSL_malloc((size_t)ctx->msi_ctx->len_msiex);
+        if (!msi_file_read(ctx->msi_ctx->msi, dse, 0, (char *)ctx->msi_ctx->p_msiex,
+                ctx->msi_ctx->len_msiex)) {
+            printf("MsiDigitalSignatureEx stream data error\n\n");
+            OPENSSL_free(indata);
+            return 0; /* FAILED */
+        }
+    }
+    OPENSSL_free(indata);
+    return 1; /* OK */
+}
+
 /*
 Local Variables:
    c-basic-offset: 4

osslsigncode.c

@@ -2717,17 +2717,15 @@ static int verify_signed_file(FILE_FORMAT_CTX *ctx, GLOBAL_OPTIONS *options)
     STACK_OF(PKCS7) *signatures = NULL;
     int detached = options->catalog ? 1 : 0;
 
-    if (!ctx->format->check_file) {
-        printf("Unsupported method: check_file\n");
-        return 1; /* FAILED */
-    }
-
-    if (!ctx->format->check_file(ctx, detached))
-        return 1; /* FAILED */
-
     if (detached) {
         GLOBAL_OPTIONS *cat_options;
         FILE_FORMAT_CTX *cat_ctx;
+
+        if (!ctx->format->is_detaching_supported || !ctx->format->is_detaching_supported()) {
+            printf("This format does not support detached PKCS#7 signature\n");
+            return 1; /* FAILED */
+        }
+        printf("Checking the specified catalog file\n\n");
         cat_options = OPENSSL_memdup(options, sizeof(GLOBAL_OPTIONS));
         if (!cat_options) {
             printf("OPENSSL_memdup error.\n");

osslsigncode.h

@@ -513,7 +513,6 @@ struct file_format_st {
     ASN1_OBJECT *(*data_blob_get) (u_char **p, int *plen, FILE_FORMAT_CTX *ctx);
     PKCS7 *(*pkcs7_contents_get) (FILE_FORMAT_CTX *ctx, BIO *hash, const EVP_MD *md);
     int (*hash_length_get) (FILE_FORMAT_CTX *ctx);
-    int (*check_file) (FILE_FORMAT_CTX *ctx, int detached);
     u_char *(*digest_calc) (FILE_FORMAT_CTX *ctx, const EVP_MD *md);
     int (*verify_digests) (FILE_FORMAT_CTX *ctx, PKCS7 *p7);
     int (*verify_indirect_data) (FILE_FORMAT_CTX *ctx, SpcAttributeTypeAndOptionalValue *obj);
@@ -526,6 +525,7 @@ struct file_format_st {
     void (*update_data_size) (FILE_FORMAT_CTX *data, BIO *outdata, PKCS7 *p7);
     BIO *(*bio_free) (BIO *hash, BIO *outdata);
     void (*ctx_cleanup) (FILE_FORMAT_CTX *ctx, BIO *hash, BIO *outdata);
+    int (*is_detaching_supported) (void);
 };
 
 /*

pe.c

@@ -46,7 +46,6 @@ static FILE_FORMAT_CTX *pe_ctx_new(GLOBAL_OPTIONS *options, BIO *hash, BIO *outd
 static ASN1_OBJECT *pe_spc_image_data_get(u_char **p, int *plen, FILE_FORMAT_CTX *ctx);
 static PKCS7 *pe_pkcs7_contents_get(FILE_FORMAT_CTX *ctx, BIO *hash, const EVP_MD *md);
 static int pe_hash_length_get(FILE_FORMAT_CTX *ctx);
-static int pe_check_file(FILE_FORMAT_CTX *ctx, int detached);
 static u_char *pe_digest_calc(FILE_FORMAT_CTX *ctx, const EVP_MD *md);
 static int pe_verify_digests(FILE_FORMAT_CTX *ctx, PKCS7 *p7);
 static int pe_verify_indirect_data(FILE_FORMAT_CTX *ctx, SpcAttributeTypeAndOptionalValue *obj);
@@ -59,13 +58,13 @@ static int pe_append_pkcs7(FILE_FORMAT_CTX *ctx, BIO *outdata, PKCS7 *p7);
 static void pe_update_data_size(FILE_FORMAT_CTX *ctx, BIO *outdata, PKCS7 *p7);
 static BIO *pe_bio_free(BIO *hash, BIO *outdata);
 static void pe_ctx_cleanup(FILE_FORMAT_CTX *ctx, BIO *hash, BIO *outdata);
+static int pe_is_detaching_supported(void);
 
 FILE_FORMAT file_format_pe = {
     .ctx_new = pe_ctx_new,
     .data_blob_get = pe_spc_image_data_get,
     .pkcs7_contents_get = pe_pkcs7_contents_get,
     .hash_length_get = pe_hash_length_get,
-    .check_file = pe_check_file,
     .digest_calc = pe_digest_calc,
     .verify_digests = pe_verify_digests,
     .verify_indirect_data = pe_verify_indirect_data,
@@ -77,7 +76,8 @@ FILE_FORMAT file_format_pe = {
     .append_pkcs7 = pe_append_pkcs7,
     .update_data_size = pe_update_data_size,
     .bio_free = pe_bio_free,
-    .ctx_cleanup = pe_ctx_cleanup
+    .ctx_cleanup = pe_ctx_cleanup,
+    .is_detaching_supported = pe_is_detaching_supported
 };
 
 /* Prototypes */
@@ -91,6 +91,7 @@ static int pe_page_hash_get(u_char **ph, int *phlen, int *phtype, SpcAttributeTy
 static u_char *pe_page_hash_calc(int *rphlen, FILE_FORMAT_CTX *ctx, int phtype);
 static int pe_verify_page_hash(FILE_FORMAT_CTX *ctx, u_char *ph, int phlen, int phtype);
 static SpcLink *pe_page_hash_link_get(FILE_FORMAT_CTX *ctx, int phtype);
+static int pe_check_file(FILE_FORMAT_CTX *ctx);
 
 
 /*
@@ -210,63 +211,6 @@ static int pe_hash_length_get(FILE_FORMAT_CTX *ctx)
     return EVP_MD_size(ctx->options->md);
 }
 
-/*
- * Print current and calculated PE checksum,
- * check if the signature exists.
- * [in, out] ctx: structure holds input and output data
- * [in] detached: embedded/detached PKCS#7 signature switch
- * [returns] 0 on error or 1 on success
- */
-static int pe_check_file(FILE_FORMAT_CTX *ctx, int detached)
-{
-    uint32_t real_pe_checksum, sum = 0;
-
-    if (!ctx) {
-        printf("Init error\n\n");
-        return 0; /* FAILED */
-    }
-    real_pe_checksum = pe_calc_realchecksum(ctx);
-    if (ctx->pe_ctx->pe_checksum == real_pe_checksum) {
-        printf("PE checksum   : %08X\n\n", real_pe_checksum);
-    } else {
-        printf("Current PE checksum   : %08X\n", ctx->pe_ctx->pe_checksum);
-        printf("Calculated PE checksum: %08X\n", real_pe_checksum);
-        printf("Warning: invalid PE checksum\n\n");
-    }
-    if (detached) {
-        printf("Checking the specified catalog file\n\n");
-        return 1; /* OK */
-    }
-    if (ctx->pe_ctx->sigpos == 0 || ctx->pe_ctx->siglen == 0
-        || ctx->pe_ctx->sigpos > ctx->pe_ctx->fileend) {
-        printf("No signature found\n\n");
-        return 0; /* FAILED */
-    }
-    /*
-     * If the sum of the rounded dwLength values does not equal the Size value,
-     * then either the attribute certificate table or the Size field is corrupted.
-     */
-    while (sum < ctx->pe_ctx->siglen) {
-        uint32_t len = GET_UINT32_LE(ctx->options->indata + ctx->pe_ctx->sigpos + sum);
-        if (ctx->pe_ctx->siglen - len > 8) {
-            printf("Corrupted attribute certificate table\n");
-            printf("Attribute certificate table size  : %08X\n", ctx->pe_ctx->siglen);
-            printf("Attribute certificate entry length: %08X\n\n", len);
-            return 0; /* FAILED */
-        }
-        /* quadword align data */
-        len += len % 8 ? 8 - len % 8 : 0;
-        sum += len;
-    }
-    if (sum != ctx->pe_ctx->siglen) {
-        printf("Corrupted attribute certificate table\n");
-        printf("Attribute certificate table size  : %08X\n", ctx->pe_ctx->siglen);
-        printf("Sum of the rounded dwLength values: %08X\n\n", sum);
-        return 0; /* FAILED */
-    }
-    return 1; /* OK */
-}
-
 /*
  * Returns a message digest value of a signed or unsigned PE file.
  * [in] ctx: structure holds input and output data
@@ -378,8 +322,7 @@ static int pe_verify_indirect_data(FILE_FORMAT_CTX *ctx, SpcAttributeTypeAndOpti
  */
 static PKCS7 *pe_pkcs7_extract(FILE_FORMAT_CTX *ctx)
 {
-    if (ctx->pe_ctx->sigpos == 0 || ctx->pe_ctx->siglen == 0
+    if (!pe_check_file(ctx)) {
-        || ctx->pe_ctx->sigpos > ctx->pe_ctx->fileend) {
         return NULL; /* FAILED */
     }
     return pe_pkcs7_get_file(ctx->options->indata, ctx->pe_ctx);
@@ -404,8 +347,7 @@ static PKCS7 *pe_pkcs7_extract_to_nest(FILE_FORMAT_CTX *ctx)
  */
 static int pe_remove_pkcs7(FILE_FORMAT_CTX *ctx, BIO *hash, BIO *outdata)
 {
-    if (ctx->pe_ctx->sigpos == 0 || ctx->pe_ctx->siglen == 0
+    if (!pe_check_file(ctx)) {
-        || ctx->pe_ctx->sigpos > ctx->pe_ctx->fileend) {
         return 1; /* FAILED, no signature */
     }
     /* Strip current signature */
@@ -581,6 +523,11 @@ static void pe_ctx_cleanup(FILE_FORMAT_CTX *ctx, BIO *hash, BIO *outdata)
     OPENSSL_free(ctx);
 }
 
+static int pe_is_detaching_supported(void)
+{
+    return 1; /* OK */
+}
+
 /*
  * PE helper functions
  */
@@ -1217,6 +1164,58 @@ static SpcLink *pe_page_hash_link_get(FILE_FORMAT_CTX *ctx, int phtype)
     return link;
 }
 
+/*
+ * Print current and calculated PE checksum,
+ * check if the signature exists.
+ * [in, out] ctx: structure holds input and output data
+ * [returns] 0 on error or 1 on success
+ */
+static int pe_check_file(FILE_FORMAT_CTX *ctx)
+{
+    uint32_t real_pe_checksum, sum = 0;
+
+    if (!ctx) {
+        printf("Init error\n\n");
+        return 0; /* FAILED */
+    }
+    real_pe_checksum = pe_calc_realchecksum(ctx);
+    if (ctx->pe_ctx->pe_checksum == real_pe_checksum) {
+        printf("PE checksum   : %08X\n\n", real_pe_checksum);
+    } else {
+        printf("Current PE checksum   : %08X\n", ctx->pe_ctx->pe_checksum);
+        printf("Calculated PE checksum: %08X\n", real_pe_checksum);
+        printf("Warning: invalid PE checksum\n\n");
+    }
+    if (ctx->pe_ctx->sigpos == 0 || ctx->pe_ctx->siglen == 0
+        || ctx->pe_ctx->sigpos > ctx->pe_ctx->fileend) {
+        printf("No signature found\n\n");
+        return 0; /* FAILED */
+    }
+    /*
+     * If the sum of the rounded dwLength values does not equal the Size value,
+     * then either the attribute certificate table or the Size field is corrupted.
+     */
+    while (sum < ctx->pe_ctx->siglen) {
+        uint32_t len = GET_UINT32_LE(ctx->options->indata + ctx->pe_ctx->sigpos + sum);
+        if (ctx->pe_ctx->siglen - len > 8) {
+            printf("Corrupted attribute certificate table\n");
+            printf("Attribute certificate table size  : %08X\n", ctx->pe_ctx->siglen);
+            printf("Attribute certificate entry length: %08X\n\n", len);
+            return 0; /* FAILED */
+        }
+        /* quadword align data */
+        len += len % 8 ? 8 - len % 8 : 0;
+        sum += len;
+    }
+    if (sum != ctx->pe_ctx->siglen) {
+        printf("Corrupted attribute certificate table\n");
+        printf("Attribute certificate table size  : %08X\n", ctx->pe_ctx->siglen);
+        printf("Sum of the rounded dwLength values: %08X\n\n", sum);
+        return 0; /* FAILED */
+    }
+    return 1; /* OK */
+}
+
 /*
 Local Variables:
    c-basic-offset: 4

script.c

@@ -54,7 +54,6 @@ static FILE_FORMAT_CTX *script_ctx_new(GLOBAL_OPTIONS *options, BIO *hash, BIO *
 static ASN1_OBJECT *script_spc_sip_info_get(u_char **p, int *plen, FILE_FORMAT_CTX *ctx);
 static PKCS7 *script_pkcs7_contents_get(FILE_FORMAT_CTX *ctx, BIO *hash, const EVP_MD *md);
 static int script_hash_length_get(FILE_FORMAT_CTX *ctx);
-static int script_check_file(FILE_FORMAT_CTX *ctx, int detached);
 static u_char *script_digest_calc(FILE_FORMAT_CTX *ctx, const EVP_MD *md);
 static int script_verify_digests(FILE_FORMAT_CTX *ctx, PKCS7 *p7);
 static PKCS7 *script_pkcs7_extract(FILE_FORMAT_CTX *ctx);
@@ -65,13 +64,13 @@ static PKCS7 *script_pkcs7_signature_new(FILE_FORMAT_CTX *ctx, BIO *hash);
 static int script_append_pkcs7(FILE_FORMAT_CTX *ctx, BIO *outdata, PKCS7 *p7);
 static BIO *script_bio_free(BIO *hash, BIO *outdata);
 static void script_ctx_cleanup(FILE_FORMAT_CTX *ctx, BIO *hash, BIO *outdata);
+static int script_is_detaching_supported(void);
 
 FILE_FORMAT file_format_script = {
     .ctx_new        = script_ctx_new,
     .data_blob_get  = script_spc_sip_info_get,
     .pkcs7_contents_get = script_pkcs7_contents_get,
     .hash_length_get = script_hash_length_get,
-    .check_file     = script_check_file,
     .digest_calc    = script_digest_calc,
     .verify_digests = script_verify_digests,
     .pkcs7_extract  = script_pkcs7_extract,
@@ -82,6 +81,7 @@ FILE_FORMAT file_format_script = {
     .append_pkcs7   = script_append_pkcs7,
     .bio_free       = script_bio_free,
     .ctx_cleanup    = script_ctx_cleanup,
+    .is_detaching_supported = script_is_detaching_supported
 };
 
 /* helper functions */
@@ -93,6 +93,7 @@ static size_t utf16_to_utf8(const uint16_t *data, size_t len, char **out_utf8);
 static BIO *script_digest_calc_bio(FILE_FORMAT_CTX *ctx, const EVP_MD *md);
 static int script_digest_convert(BIO *hash, FILE_FORMAT_CTX *ctx, size_t len);
 static int script_write_bio(BIO *data, char *indata, size_t len);
+static int script_check_file(FILE_FORMAT_CTX *ctx);
 
 /*
  * Allocate and return a script file format context.
@@ -229,32 +230,6 @@ static int script_hash_length_get(FILE_FORMAT_CTX *ctx)
     return EVP_MD_size(ctx->options->md);
 }
 
-/*
- * Check if the signature exists.
- * FIXME: check it in pkcs7_extract()
- * [in, out] ctx: structure holds input and output data
- * [in] detached: embedded/detached PKCS#7 signature switch
- * [returns] 0 on error or 1 on success
- */
-static int script_check_file(FILE_FORMAT_CTX *ctx, int detached)
-{
-    if (!ctx) {
-        printf("Init error\n\n");
-        return 0; /* FAILED */
-    }
-    if (detached) {
-        printf("Checking the specified catalog file\n\n");
-        return 1; /* OK */
-    }
-    if (ctx->script_ctx->sigpos == 0
-        || ctx->script_ctx->sigpos > ctx->script_ctx->fileend) {
-        printf("No signature found\n\n");
-        return 0; /* FAILED */
-    }
-
-    return 1; /* OK */
-}
-
 /*
  * Compute a simple sha1/sha256 message digest of the MSI file
  * for use with a catalog file.
@@ -357,6 +332,9 @@ static PKCS7 *script_pkcs7_extract(FILE_FORMAT_CTX *ctx)
     size_t signature_footer_len = strlen(signature_footer);
     PKCS7 *retval = NULL;
 
+    if (!script_check_file(ctx)) {
+        return NULL; /* FAILED, no signature */
+    }
     /* extract Base64 signature */
     if (ctx->script_ctx->utf == 8) {
         base64_len = signature_len;
@@ -470,8 +448,7 @@ static int script_remove_pkcs7(FILE_FORMAT_CTX *ctx, BIO *hash, BIO *outdata)
 {
     /* squash the unused parameter warning */
     (void)hash;
-    if (ctx->script_ctx->sigpos == 0
+    if (!script_check_file(ctx)) {
-        || ctx->script_ctx->sigpos > ctx->script_ctx->fileend) {
         return 1; /* FAILED, no signature */
     }
     if (!script_write_bio(outdata, ctx->options->indata, ctx->script_ctx->sigpos)) {
@@ -624,6 +601,11 @@ static void script_ctx_cleanup(FILE_FORMAT_CTX *ctx, BIO *hash, BIO *outdata)
     OPENSSL_free(ctx);
 }
 
+static int script_is_detaching_supported(void)
+{
+    return 1; /* OK */
+}
+
 /*
  * Script helper functions
  */
@@ -884,6 +866,26 @@ static int script_write_bio(BIO *bio, char *indata, size_t len)
     return 1; /* OK */
 }
 
+/*
+ * Check if the signature exists.
+ * [in, out] ctx: structure holds input and output data
+ * [returns] 0 on error or 1 on success
+ */
+static int script_check_file(FILE_FORMAT_CTX *ctx)
+{
+    if (!ctx) {
+        printf("Init error\n\n");
+        return 0; /* FAILED */
+    }
+    if (ctx->script_ctx->sigpos == 0
+        || ctx->script_ctx->sigpos > ctx->script_ctx->fileend) {
+        printf("No signature found\n\n");
+        return 0; /* FAILED */
+    }
+
+    return 1; /* OK */
+}
+
 /*
 Local Variables:
    c-basic-offset: 4