Handle sign/digest API call order

Change-Id: Idf64e1b9c3d6dbbe319dbce54786407c941db90a

4 files changed, 111 insertions, 37 deletions

diff --git a/src/digest.c b/src/digest.c
index 361b9f8..5b4b2f4 100644
--- a/src/digest.c
+++ b/src/digest.c
@@ -49,8 +49,23 @@ struct yaca_digest_context_s {
 	struct yaca_context_s ctx;
 
 	EVP_MD_CTX *md_ctx;
+	enum context_state_e state;
 };
 
+static bool CTX_DEFAULT_STATES[CTX_COUNT][CTX_COUNT] = {
+/* from \ to  INIT, MSG, FIN */
+/* INIT */  { 0,    1,    1 },
+/* MSG  */  { 0,    1,    1 },
+/* FIN  */  { 0,    0,    0 },
+};
+
+static bool verify_state_change(struct yaca_digest_context_s *c, enum context_state_e to)
+{
+	int from = c->state;
+
+	return CTX_DEFAULT_STATES[from][to];
+}
+
 static struct yaca_digest_context_s *get_digest_context(const yaca_context_h ctx)
 {
 	if (ctx == YACA_CONTEXT_NULL)
@@ -136,6 +151,8 @@ API int yaca_digest_initialize(yaca_context_h *ctx, yaca_digest_algorithm_e algo
 	nc->ctx.type = YACA_CONTEXT_DIGEST;
 	nc->ctx.context_destroy = destroy_digest_context;
 	nc->ctx.get_output_length = get_digest_output_length;
+	nc->ctx.set_property = NULL;
+	nc->ctx.get_property = NULL;
 
 	ret = digest_get_algorithm(algo, &md);
 	if (ret != YACA_ERROR_NONE)
@@ -155,6 +172,7 @@ API int yaca_digest_initialize(yaca_context_h *ctx, yaca_digest_algorithm_e algo
 		goto exit;
 	}
 
+	nc->state = CTX_INITIALIZED;
 	*ctx = (yaca_context_h)nc;
 	nc = NULL;
 	ret = YACA_ERROR_NONE;
@@ -173,6 +191,9 @@ API int yaca_digest_update(yaca_context_h ctx, const char *message, size_t messa
 	if (c == NULL || message == NULL || message_len == 0)
 		return YACA_ERROR_INVALID_PARAMETER;
 
+	if (!verify_state_change(c, CTX_MSG_UPDATED))
+		return YACA_ERROR_INVALID_PARAMETER;
+
 	ret = EVP_DigestUpdate(c->md_ctx, message, message_len);
 	if (ret != 1) {
 		ret = YACA_ERROR_INTERNAL;
@@ -180,6 +201,7 @@ API int yaca_digest_update(yaca_context_h ctx, const char *message, size_t messa
 		return ret;
 	}
 
+	c->state = CTX_MSG_UPDATED;
 	return YACA_ERROR_NONE;
 }
 
@@ -192,6 +214,9 @@ API int yaca_digest_finalize(yaca_context_h ctx, char *digest, size_t *digest_le
 	if (c == NULL || digest == NULL || digest_len == NULL)
 		return YACA_ERROR_INVALID_PARAMETER;
 
+	if (!verify_state_change(c, CTX_FINALIZED))
+		return YACA_ERROR_INVALID_PARAMETER;
+
 	if (*digest_len == 0 || *digest_len > UINT_MAX) /* DigestFinal accepts UINT */
 		return YACA_ERROR_INVALID_PARAMETER;
 
@@ -202,6 +227,7 @@ API int yaca_digest_finalize(yaca_context_h ctx, char *digest, size_t *digest_le
 		return ret;
 	}
 
+	c->state = CTX_FINALIZED;
 	*digest_len = len;
 
 	return YACA_ERROR_NONE;
diff --git a/src/encrypt.c b/src/encrypt.c
index 9bf4e74..1c3bcf5 100644
--- a/src/encrypt.c
+++ b/src/encrypt.c
@@ -117,7 +117,7 @@ static bool is_encryption_op(enum encrypt_op_type_e op_type)
 	return (op_type == OP_ENCRYPT || op_type == OP_SEAL);
 }
 
-static bool DEFAULT_STATES[STATE_COUNT][STATE_COUNT] = {
+static bool DEFAULT_STATES[ENC_CTX_COUNT][ENC_CTX_COUNT] = {
 /* from \ to  INIT, MLEN, AAD,  MSG,  TAG,  TLEN, FIN */
 /* INIT */  { 0,    0,    0,    1,    0,    0,    1 },
 /* MLEN  */ { 0,    0,    0,    0,    0,    0,    0 },
@@ -128,7 +128,7 @@ static bool DEFAULT_STATES[STATE_COUNT][STATE_COUNT] = {
 /* FIN  */  { 0,    0,    0,    0,    0,    0,    0 },
 };
 
-static bool GCM_STATES[2][STATE_COUNT][STATE_COUNT] = { {
+static bool GCM_STATES[2][ENC_CTX_COUNT][ENC_CTX_COUNT] = { {
 /* ENCRYPTION */
 /* from \ to  INIT, MLEN, AAD,  MSG,  TAG,  TLEN, FIN */
 /* INIT */  { 0,    0,    1,    1,    0,    0,    1 },
@@ -150,7 +150,7 @@ static bool GCM_STATES[2][STATE_COUNT][STATE_COUNT] = { {
 /* FIN  */  { 0,    0,    0,    0,    0,    0,    0 },
 } };
 
-static bool CCM_STATES[2][STATE_COUNT][STATE_COUNT] = { {
+static bool CCM_STATES[2][ENC_CTX_COUNT][ENC_CTX_COUNT] = { {
 /* ENCRYPTION */
 /* from \ to  INIT, MLEN, AAD,  MSG,  TAG,  TLEN, FIN */
 /* INIT */  { 0,    1,    0,    1,    0,    1,    0 },
@@ -172,7 +172,7 @@ static bool CCM_STATES[2][STATE_COUNT][STATE_COUNT] = { {
 /* FIN  */  { 0,    0,    0,    0,    0,    0,    0 },
 } };
 
-static bool WRAP_STATES[STATE_COUNT][STATE_COUNT] = {
+static bool WRAP_STATES[ENC_CTX_COUNT][ENC_CTX_COUNT] = {
 /* from \ to  INIT, MLEN, AAD,  MSG,  TAG,  TLEN, FIN */
 /* INIT */  { 0,    0,    0,    1,    0,    0,    0 },
 /* MLEN */  { 0,    0,    0,    0,    0,    0,    0 },
@@ -668,79 +668,79 @@ int set_encrypt_property(yaca_context_h ctx,
 	switch (property) {
 	case YACA_PROPERTY_GCM_AAD:
 		if (mode != EVP_CIPH_GCM_MODE ||
-		    !verify_state_change(c, STATE_AAD_UPDATED))
+		    !verify_state_change(c, ENC_CTX_AAD_UPDATED))
 			return YACA_ERROR_INVALID_PARAMETER;
 
 		if (EVP_CipherUpdate(c->cipher_ctx, NULL, &len, value, value_len) != 1) {
 			ERROR_DUMP(YACA_ERROR_INTERNAL);
 			return YACA_ERROR_INTERNAL;
 		}
-		c->state = STATE_AAD_UPDATED;
+		c->state = ENC_CTX_AAD_UPDATED;
 		break;
 	case YACA_PROPERTY_CCM_AAD:
 		if (mode != EVP_CIPH_CCM_MODE ||
-		    !verify_state_change(c, STATE_AAD_UPDATED))
+		    !verify_state_change(c, ENC_CTX_AAD_UPDATED))
 			return YACA_ERROR_INVALID_PARAMETER;
 
 		if (EVP_CipherUpdate(c->cipher_ctx, NULL, &len, value, value_len) != 1) {
 			ERROR_DUMP(YACA_ERROR_INTERNAL);
 			return YACA_ERROR_INTERNAL;
 		}
-		c->state = STATE_AAD_UPDATED;
+		c->state = ENC_CTX_AAD_UPDATED;
 		break;
 	case YACA_PROPERTY_GCM_TAG:
 		if (mode != EVP_CIPH_GCM_MODE || is_encryption_op(c->op_type) ||
 		    !is_valid_tag_len(mode, value_len) ||
-		    !verify_state_change(c, STATE_TAG_SET))
+		    !verify_state_change(c, ENC_CTX_TAG_SET))
 			return YACA_ERROR_INVALID_PARAMETER;
 
 		if (EVP_CIPHER_CTX_ctrl(c->cipher_ctx, EVP_CTRL_GCM_SET_TAG, value_len, (void*)value) != 1) {
 			ERROR_DUMP(YACA_ERROR_INTERNAL);
 			return YACA_ERROR_INTERNAL;
 		}
-		c->state = STATE_TAG_SET;
+		c->state = ENC_CTX_TAG_SET;
 		break;
 	case YACA_PROPERTY_GCM_TAG_LEN:
 		if (value_len != sizeof(size_t) || mode != EVP_CIPH_GCM_MODE ||
 		    !is_encryption_op(c->op_type) ||
 		    !is_valid_tag_len(mode, *(size_t*)value) ||
-		    !verify_state_change(c, STATE_TAG_LENGTH_SET))
+		    !verify_state_change(c, ENC_CTX_TAG_LENGTH_SET))
 			return YACA_ERROR_INVALID_PARAMETER;
 
 		c->tag_len = *(size_t*)value;
-		c->state = STATE_TAG_LENGTH_SET;
+		c->state = ENC_CTX_TAG_LENGTH_SET;
 		break;
 	case YACA_PROPERTY_CCM_TAG:
 		if (mode != EVP_CIPH_CCM_MODE || is_encryption_op(c->op_type) ||
 		    !is_valid_tag_len(mode, value_len) ||
-		    !verify_state_change(c, STATE_TAG_SET))
+		    !verify_state_change(c, ENC_CTX_TAG_SET))
 			return YACA_ERROR_INVALID_PARAMETER;
 
 		ret = encrypt_ctx_set_ccm_tag(c, (char*)value, value_len);
 		if (ret != YACA_ERROR_NONE)
 			return ret;
 
-		c->state = STATE_TAG_SET;
+		c->state = ENC_CTX_TAG_SET;
 		break;
 	case YACA_PROPERTY_CCM_TAG_LEN:
 		if (value_len != sizeof(size_t) || mode != EVP_CIPH_CCM_MODE ||
 		    !is_encryption_op(c->op_type) ||
 		    !is_valid_tag_len(mode, *(size_t*)value) ||
-		    !verify_state_change(c, STATE_TAG_LENGTH_SET))
+		    !verify_state_change(c, ENC_CTX_TAG_LENGTH_SET))
 			return YACA_ERROR_INVALID_PARAMETER;
 
 		ret = encrypt_ctx_set_ccm_tag_len(c, *(size_t*)value);
 		if (ret != YACA_ERROR_NONE)
 			return ret;
 
-		c->state = STATE_TAG_LENGTH_SET;
+		c->state = ENC_CTX_TAG_LENGTH_SET;
 		break;
 	case YACA_PROPERTY_PADDING:
 		if ((mode != EVP_CIPH_ECB_MODE && mode != EVP_CIPH_CBC_MODE) ||
 		    value_len != sizeof(yaca_padding_e) ||
 		    (*(yaca_padding_e*)value != YACA_PADDING_NONE &&
 		    *(yaca_padding_e*)value != YACA_PADDING_PKCS7) ||
-		    c->state == STATE_FINALIZED)
+		    c->state == ENC_CTX_FINALIZED)
 			return YACA_ERROR_INVALID_PARAMETER;
 
 		int padding = *(yaca_padding_e*)value == YACA_PADDING_NONE ? 0 : 1;
@@ -754,7 +754,7 @@ int set_encrypt_property(yaca_context_h ctx,
 	case YACA_PROPERTY_RC2_EFFECTIVE_KEY_BITS:
 		if (value_len != sizeof(size_t) ||
 		    (nid != NID_rc2_cbc && nid != NID_rc2_ecb && nid != NID_rc2_cfb64 && nid != NID_rc2_ofb64) ||
-		    c->state != STATE_INITIALIZED)
+		    c->state != ENC_CTX_INITIALIZED)
 			return YACA_ERROR_INVALID_PARAMETER;
 
 		ret = encrypt_ctx_set_rc2_effective_key_bits(c, *(size_t*)value);
@@ -783,7 +783,7 @@ int get_encrypt_property(const yaca_context_h ctx, yaca_property_e property,
 		if (value_len == NULL ||
 		    !is_encryption_op(c->op_type) ||
 		    mode != EVP_CIPH_GCM_MODE ||
-		    (c->state != STATE_TAG_LENGTH_SET && c->state != STATE_FINALIZED))
+		    (c->state != ENC_CTX_TAG_LENGTH_SET && c->state != ENC_CTX_FINALIZED))
 			return YACA_ERROR_INVALID_PARAMETER;
 
 		assert(c->tag_len <= INT_MAX);
@@ -801,7 +801,7 @@ int get_encrypt_property(const yaca_context_h ctx, yaca_property_e property,
 		if (value_len == NULL ||
 		    !is_encryption_op(c->op_type) ||
 		    mode != EVP_CIPH_CCM_MODE ||
-		    c->state != STATE_FINALIZED)
+		    c->state != ENC_CTX_FINALIZED)
 			return YACA_ERROR_INVALID_PARAMETER;
 
 		assert(c->tag_len <= INT_MAX);
@@ -946,7 +946,7 @@ int encrypt_initialize(yaca_context_h *ctx,
 			goto exit;
 	}
 
-	nc->state = STATE_INITIALIZED;
+	nc->state = ENC_CTX_INITIALIZED;
 
 	*ctx = (yaca_context_h)nc;
 	nc = NULL;
@@ -977,13 +977,13 @@ int encrypt_update(yaca_context_h ctx,
 
 	enum encrypt_context_state_e target_state;
 	if (output == NULL && input == NULL)
-		target_state = STATE_MSG_LENGTH_UPDATED;
+		target_state = ENC_CTX_MSG_LENGTH_UPDATED;
 	else if (output == NULL)
-		target_state = STATE_AAD_UPDATED;
+		target_state = ENC_CTX_AAD_UPDATED;
 	else if (input == NULL)
 		return YACA_ERROR_INVALID_PARAMETER;
 	else
-		target_state = STATE_MSG_UPDATED;
+		target_state = ENC_CTX_MSG_UPDATED;
 
 	if (!verify_state_change(c, target_state))
 		return YACA_ERROR_INVALID_PARAMETER;
@@ -1065,7 +1065,7 @@ int encrypt_finalize(yaca_context_h ctx,
 	if (c == NULL || output == NULL || output_len == NULL || op_type != c->op_type)
 		return YACA_ERROR_INVALID_PARAMETER;
 
-	if (!verify_state_change(c, STATE_FINALIZED))
+	if (!verify_state_change(c, ENC_CTX_FINALIZED))
 		return YACA_ERROR_INVALID_PARAMETER;
 
 	mode = EVP_CIPHER_CTX_mode(c->cipher_ctx);
@@ -1089,7 +1089,7 @@ int encrypt_finalize(yaca_context_h ctx,
 	if (EVP_CIPHER_CTX_test_flags(c->cipher_ctx, EVP_CIPH_FLAG_LENGTH_BITS) != 0)
 		*output_len /= 8;
 
-	c->state = STATE_FINALIZED;
+	c->state = ENC_CTX_FINALIZED;
 	return YACA_ERROR_NONE;
 }
 
diff --git a/src/internal.h b/src/internal.h
index 20da604..4bae2aa 100644
--- a/src/internal.h
+++ b/src/internal.h
@@ -98,15 +98,23 @@ struct yaca_backup_context_s {
 };
 
 enum encrypt_context_state_e {
-	STATE_INITIALIZED = 0,
-	STATE_MSG_LENGTH_UPDATED,
-	STATE_AAD_UPDATED,
-	STATE_MSG_UPDATED,
-	STATE_TAG_SET,
-	STATE_TAG_LENGTH_SET,
-	STATE_FINALIZED,
-
-	STATE_COUNT,
+	ENC_CTX_INITIALIZED = 0,
+	ENC_CTX_MSG_LENGTH_UPDATED,
+	ENC_CTX_AAD_UPDATED,
+	ENC_CTX_MSG_UPDATED,
+	ENC_CTX_TAG_SET,
+	ENC_CTX_TAG_LENGTH_SET,
+	ENC_CTX_FINALIZED,
+
+	ENC_CTX_COUNT,
+};
+
+enum context_state_e {
+	CTX_INITIALIZED = 0,
+	CTX_MSG_UPDATED,
+	CTX_FINALIZED,
+
+	CTX_COUNT,
 };
 
 struct yaca_encrypt_context_s {
diff --git a/src/sign.c b/src/sign.c
index 4aba130..15f0ba9 100644
--- a/src/sign.c
+++ b/src/sign.c
@@ -48,8 +48,23 @@ struct yaca_sign_context_s {
 
 	EVP_MD_CTX *md_ctx;
 	enum sign_op_type op_type;
+	enum context_state_e state;
 };
 
+static bool CTX_DEFAULT_STATES[CTX_COUNT][CTX_COUNT] = {
+/* from \ to  INIT, MSG, FIN */
+/* INIT */  { 0,    1,    1 },
+/* MSG  */  { 0,    1,    1 },
+/* FIN  */  { 0,    0,    0 },
+};
+
+static bool verify_state_change(struct yaca_sign_context_s *c, enum context_state_e to)
+{
+	int from = c->state;
+
+	return CTX_DEFAULT_STATES[from][to];
+}
+
 static struct yaca_sign_context_s *get_sign_context(const yaca_context_h ctx)
 {
 	if (ctx == YACA_CONTEXT_NULL)
@@ -120,7 +135,7 @@ int set_sign_property(yaca_context_h ctx,
 	EVP_PKEY *pkey;
 	EVP_PKEY_CTX *pctx;
 
-	if (c == NULL || value == NULL)
+	if (c == NULL || value == NULL || c->state == CTX_FINALIZED)
 		return YACA_ERROR_INVALID_PARAMETER;
 
 	assert(c->md_ctx != NULL);
@@ -218,6 +233,7 @@ API int yaca_sign_initialize(yaca_context_h *ctx,
 		goto exit;
 	}
 
+	nc->state = CTX_INITIALIZED;
 	*ctx = (yaca_context_h)nc;
 	nc = NULL;
 	ret = YACA_ERROR_NONE;
@@ -250,6 +266,8 @@ API int yaca_sign_initialize_hmac(yaca_context_h *ctx,
 	nc->ctx.type = YACA_CONTEXT_SIGN;
 	nc->ctx.context_destroy = destroy_sign_context;
 	nc->ctx.get_output_length = get_sign_output_length;
+	nc->ctx.set_property = NULL;
+	nc->ctx.get_property = NULL;
 
 	pkey = EVP_PKEY_new_mac_key(EVP_PKEY_HMAC,
 	                            NULL,
@@ -279,6 +297,7 @@ API int yaca_sign_initialize_hmac(yaca_context_h *ctx,
 		goto exit;
 	}
 
+	nc->state = CTX_INITIALIZED;
 	*ctx = (yaca_context_h)nc;
 	nc = NULL;
 	ret = YACA_ERROR_NONE;
@@ -313,6 +332,8 @@ API int yaca_sign_initialize_cmac(yaca_context_h *ctx,
 	nc->ctx.type = YACA_CONTEXT_SIGN;
 	nc->ctx.context_destroy = destroy_sign_context;
 	nc->ctx.get_output_length = get_sign_output_length;
+	nc->ctx.set_property = NULL;
+	nc->ctx.get_property = NULL;
 
 	ret = encrypt_get_algorithm(algo, YACA_BCM_CBC, simple_key->bit_len, &cipher);
 	if (ret != YACA_ERROR_NONE)
@@ -361,6 +382,7 @@ API int yaca_sign_initialize_cmac(yaca_context_h *ctx,
 		goto exit;
 	}
 
+	nc->state = CTX_INITIALIZED;
 	*ctx = (yaca_context_h)nc;
 	nc = NULL;
 	ret = YACA_ERROR_NONE;
@@ -384,6 +406,9 @@ API int yaca_sign_update(yaca_context_h ctx,
 	    message == NULL || message_len == 0)
 		return YACA_ERROR_INVALID_PARAMETER;
 
+	if (!verify_state_change(c, CTX_MSG_UPDATED))
+		return YACA_ERROR_INVALID_PARAMETER;
+
 	ret = EVP_DigestSignUpdate(c->md_ctx, message, message_len);
 	if (ret != 1) {
 		ret = YACA_ERROR_INTERNAL;
@@ -391,6 +416,7 @@ API int yaca_sign_update(yaca_context_h ctx,
 		return ret;
 	}
 
+	c->state = CTX_MSG_UPDATED;
 	return YACA_ERROR_NONE;
 }
 
@@ -405,6 +431,9 @@ API int yaca_sign_finalize(yaca_context_h ctx,
 	    signature == NULL || signature_len == NULL || *signature_len == 0)
 		return YACA_ERROR_INVALID_PARAMETER;
 
+	if (!verify_state_change(c, CTX_FINALIZED))
+		return YACA_ERROR_INVALID_PARAMETER;
+
 	ret = EVP_DigestSignFinal(c->md_ctx, (unsigned char *)signature, signature_len);
 	if (ret != 1) {
 		ret = YACA_ERROR_INTERNAL;
@@ -412,6 +441,7 @@ API int yaca_sign_finalize(yaca_context_h ctx,
 		return ret;
 	}
 
+	c->state = CTX_FINALIZED;
 	return YACA_ERROR_NONE;
 }
 
@@ -465,6 +495,7 @@ API int yaca_verify_initialize(yaca_context_h *ctx,
 		goto exit;
 	}
 
+	nc->state = CTX_INITIALIZED;
 	*ctx = (yaca_context_h)nc;
 	nc = NULL;
 	ret = YACA_ERROR_NONE;
@@ -485,6 +516,9 @@ API int yaca_verify_update(yaca_context_h ctx,
 	if (c == NULL || message == NULL || message_len == 0 || c->op_type != OP_VERIFY)
 		return YACA_ERROR_INVALID_PARAMETER;
 
+	if (!verify_state_change(c, CTX_MSG_UPDATED))
+		return YACA_ERROR_INVALID_PARAMETER;
+
 	ret = EVP_DigestVerifyUpdate(c->md_ctx, message, message_len);
 	if (ret != 1) {
 		ret = YACA_ERROR_INTERNAL;
@@ -492,6 +526,7 @@ API int yaca_verify_update(yaca_context_h ctx,
 		return ret;
 	}
 
+	c->state = CTX_MSG_UPDATED;
 	return YACA_ERROR_NONE;
 }
 
@@ -505,12 +540,17 @@ API int yaca_verify_finalize(yaca_context_h ctx,
 	if (c == NULL || signature == NULL || signature_len == 0 || c->op_type != OP_VERIFY)
 		return YACA_ERROR_INVALID_PARAMETER;
 
+	if (!verify_state_change(c, CTX_FINALIZED))
+		return YACA_ERROR_INVALID_PARAMETER;
+
 	ret = EVP_DigestVerifyFinal(c->md_ctx,
 	                            (unsigned char *)signature,
 	                            signature_len);
 
-	if (ret == 1)
+	if (ret == 1) {
+		c->state = CTX_FINALIZED;
 		return YACA_ERROR_NONE;
+	}
 
 	if (ret == 0) {
 		ERROR_CLEAR();