diff options
author | LonelyFool <2655294004@qq.com> | 2021-07-18 05:25:46 +0000 |
---|---|---|
committer | LonelyFool <2655294004@qq.com> | 2021-07-18 05:25:46 +0000 |
commit | dd9a794fe648878988cfe0ac3edeb92f36e3f0f5 (patch) | |
tree | d11ad502bd279b532df8bf10bfb7c6a1363885a5 /lib/libjsonpb | |
parent | fd8dc444bd8338643c2c07b435125cbf748af255 (diff) | |
download | dynpart-tools-dd9a794fe648878988cfe0ac3edeb92f36e3f0f5.tar.gz dynpart-tools-dd9a794fe648878988cfe0ac3edeb92f36e3f0f5.tar.bz2 dynpart-tools-dd9a794fe648878988cfe0ac3edeb92f36e3f0f5.zip |
super.img tools
Diffstat (limited to 'lib/libjsonpb')
-rwxr-xr-x | lib/libjsonpb/README.md | 107 | ||||
-rwxr-xr-x | lib/libjsonpb/TEST_MAPPING | 8 | ||||
-rwxr-xr-x | lib/libjsonpb/parse/Android.bp | 43 | ||||
-rwxr-xr-x | lib/libjsonpb/parse/include/jsonpb/error_or.h | 72 | ||||
-rwxr-xr-x | lib/libjsonpb/parse/include/jsonpb/jsonpb.h | 60 | ||||
-rwxr-xr-x | lib/libjsonpb/parse/jsonpb.cpp | 74 | ||||
-rwxr-xr-x | lib/libjsonpb/verify/Android.bp | 70 | ||||
-rwxr-xr-x | lib/libjsonpb/verify/include/jsonpb/json_schema_test.h | 121 | ||||
-rwxr-xr-x | lib/libjsonpb/verify/include/jsonpb/verify.h | 93 | ||||
-rwxr-xr-x | lib/libjsonpb/verify/test.cpp | 302 | ||||
-rwxr-xr-x | lib/libjsonpb/verify/test.proto | 65 | ||||
-rwxr-xr-x | lib/libjsonpb/verify/verify.cpp | 221 |
12 files changed, 1236 insertions, 0 deletions
diff --git a/lib/libjsonpb/README.md b/lib/libjsonpb/README.md new file mode 100755 index 0000000..5562c8f --- /dev/null +++ b/lib/libjsonpb/README.md @@ -0,0 +1,107 @@ +# `libjsonpbparse` + +This library provides functions to parse a JSON file to a structured Protobuf +message. + +At this time of writing, `libprotobuf-cpp-full` is at version 3.0.0-beta, and +unknown fields in a JSON file cannot be ignored. Do **NOT** use this library in +vendor / recovery until `libprotobuf-cpp-full` is updated. + +## Using `libjsoncpp` in parser code + +Since `libjsonpbparse` cannot be used in vendor / recovery processes yet, +`libjsoncpp` is used instead. However, there are notable differences in the +logic of `libjsoncpp` and `libprotobuf` when parsing JSON files. + +- There are no implicit string to integer conversion in `libjsoncpp`. Hence: + - If the Protobuf schema uses 64-bit integers (`(s|fixed|u|)int64`): + - The JSON file must use strings (to pass tests in `libjsonpbverify`) + - Parser code (that uses `libjsoncpp`) must explicitly convert strings to + integers. Example: + ```c++ + strtoull(value.asString(), 0, 10) + ``` + - If the Protobuf schema uses special floating point values: + - The JSON file must use strings (e.g. `"NaN"`, `"Infinity"`, `"-Infinity"`) + - Parser code must explicitly handle these cases. Example: + ```c++ + double d; + if (value.isNumeric()) { + d = value.asDouble(); + } else { + auto&& s = value.asString(); + if (s == "NaN") d = std::numeric_limits<double>::quiet_NaN(); + else if (s == "Infinity") d = std::numeric_limits<double>::infinity(); + else if (s == "-Infinity") d = -std::numeric_limits<double>::infinity(); + } + ``` +- `libprotobuf` accepts either `lowerCamelCase` (or `json_name` option if it is + defined) or the original field name as keys in the input JSON file. + The test in `libjsonpbverify` explicitly check this case to avoid ambiguity; + only the original field name (or `json_name` option if it is defined) can be + used. + +Once `libprotobuf` in the source tree is updated to a higher version and +`libjsonpbparse` is updated to ignore unknown fields in JSON files, all parsing +code must be converted to use `libjsonpbparse` for consistency. + +# `libjsonpbverify` + +This library provides functions and tests to examine a JSON file and validate +it against a Protobuf message definition. + +In addition to a sanity check that `libprotobuf` can convert the JSON file to a +Protobuf message (using `libjsonpbparse`), it also checks the following: + +- Whether there are fields unknown to the schema. All fields in the JSON file + must be well defined in the schema. +- Whether the Protobuf file defines JSON keys clearly. The JSON keys must be + the `json_name` option of a Protobuf field, or name of a Protobuf field if + `json_name` is not defined. `lowerCamelCase` supported by `libprotobuf` is + explicitly disallowed (unless explicitly used in `json_name`). For example, + in the following Protobuf file, only keys `foo_bar` and `barBaz` are allowed + in the JSON file: + ``` + message Foo { + string foo_bar = 1; + string bar_baz = 2 [json_name = "barBaz"]; + } + ``` +- Whether `json == convert_to_json(convert_to_pb(json))`, using `libprotobuf`. + This imposes additional restrictions including: + - Enum values must be present as names (not integer values) in the JSON file. + - 64-bit integers and special floating point values (infinity, NaN) must + always be strings. + +## Defining a JSON schema using Protobuf + +Check [JSON Mapping](https://developers.google.com/protocol-buffers/docs/proto3#json) +before defining a Protobuf object as a JSON schema. In general: + +- **Use proto3**. `libjsonverify` does not support proto2. +- JSON booleans should be `bool`. +- JSON numbers should be `(s|fixed|u|)int32`, `float`, or `double` in the schema +- JSON strings are generally `string`s, but if you want to impose more + restrictions on the string, you can also use `Timestamp`, `bytes`, + **`float`** or **`double`** (if NaN and infinity are valid values), + enumerations, etc. + - If a custom enumeration is used, parser code should **NOT** error when the + enumeration value name is unknown, as enumeration definitions may be + extended in the future. +- JSON arrays should be repeated fields. +- JSON objects should be a well-defined `message`, unless you have a good reason + to use `map<string, T>`. +- Don't use `Any`; it defeats the purpose of having the schema. + +## Validating a JSON file against a Protobuf definition + +Example: +```c++ +#include <jsonpb/verify.h> +using namespace ::android::jsonpb; +std::unique_ptr<JsonSchemaTestConfig> CreateCgroupsParam() { + +} +INSTANTIATE_TEST_SUITE_P(LibProcessgroupProto, JsonSchemaTest, + ::testing::Values(MakeTestParam<Cgroups>("cgroups.json"))); +``` diff --git a/lib/libjsonpb/TEST_MAPPING b/lib/libjsonpb/TEST_MAPPING new file mode 100755 index 0000000..69e5a25 --- /dev/null +++ b/lib/libjsonpb/TEST_MAPPING @@ -0,0 +1,8 @@ +{ + "presubmit": [ + { + "name": "libjsonpbverify_test", + "host": true + } + ] +} diff --git a/lib/libjsonpb/parse/Android.bp b/lib/libjsonpb/parse/Android.bp new file mode 100755 index 0000000..eaec342 --- /dev/null +++ b/lib/libjsonpb/parse/Android.bp @@ -0,0 +1,43 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// A convenient library to convert any JSON string to a specific Protobuf +// message using reflection. + +cc_library_static { + name: "libjsonpbparse", + host_supported: true, + + // DO NOT make it vendor_available / recovery_available; it doesn't know + // how to ignore unknown fields yet. Use it only for testing purposes. + // TODO(b/123664216): Make it understand unknown fields when libprotobuf is + // updated to version 3.1+, and let libprocessgroup to use this instead of + // libjsoncpp. + vendor_available: false, + recovery_available: false, + + export_include_dirs: ["include"], + cflags: [ + "-Wall", + "-Werror", + "-Wno-unused-parameter", + ], + srcs: [ + "jsonpb.cpp", + ], + shared_libs: [ + "libbase", + "libprotobuf-cpp-full", + ], +} diff --git a/lib/libjsonpb/parse/include/jsonpb/error_or.h b/lib/libjsonpb/parse/include/jsonpb/error_or.h new file mode 100755 index 0000000..66e2296 --- /dev/null +++ b/lib/libjsonpb/parse/include/jsonpb/error_or.h @@ -0,0 +1,72 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +#pragma once + +#include <string> +#include <variant> + +#include <android-base/logging.h> + +namespace android { +namespace jsonpb { + +template <typename T> +struct ErrorOr { + template <class... Args> + explicit ErrorOr(Args&&... args) : data_(kIndex1, std::forward<Args>(args)...) {} + T& operator*() { + CHECK(ok()); + return *std::get_if<1u>(&data_); + } + const T& operator*() const { + CHECK(ok()); + return *std::get_if<1u>(&data_); + } + T* operator->() { + CHECK(ok()); + return std::get_if<1u>(&data_); + } + const T* operator->() const { + CHECK(ok()); + return std::get_if<1u>(&data_); + } + const std::string& error() const { + CHECK(!ok()); + return *std::get_if<0u>(&data_); + } + bool ok() const { return data_.index() != 0; } + static ErrorOr<T> MakeError(const std::string& message) { + return ErrorOr<T>(message, Tag::kDummy); + } + + private: + enum class Tag { kDummy }; + static constexpr std::in_place_index_t<0> kIndex0{}; + static constexpr std::in_place_index_t<1> kIndex1{}; + ErrorOr(const std::string& msg, Tag) : data_(kIndex0, msg) {} + + std::variant<std::string, T> data_; +}; + +template <typename T> +inline ErrorOr<T> MakeError(const std::string& message) { + return ErrorOr<T>::MakeError(message); +} + +} // namespace jsonpb +} // namespace android diff --git a/lib/libjsonpb/parse/include/jsonpb/jsonpb.h b/lib/libjsonpb/parse/include/jsonpb/jsonpb.h new file mode 100755 index 0000000..350db7f --- /dev/null +++ b/lib/libjsonpb/parse/include/jsonpb/jsonpb.h @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +#pragma once + +#include <string> + +#include <jsonpb/error_or.h> + +#include <google/protobuf/message.h> + +namespace android { +namespace jsonpb { + +namespace internal { +ErrorOr<std::monostate> JsonStringToMessage(const std::string& content, + google::protobuf::Message* message); +} // namespace internal + +// TODO: JsonStringToMessage is a newly added function in protobuf +// and is not yet available in the android tree. Replace this function with +// https://developers.google.com/protocol-buffers/docs/reference/cpp/google.protobuf.util.json_util#JsonStringToMessage.details +// when the android tree gets updated +template <typename T> +ErrorOr<T> JsonStringToMessage(const std::string& content) { + ErrorOr<T> ret; + auto error = internal::JsonStringToMessage(content, &*ret); + if (!error.ok()) { + return MakeError<T>(error.error()); + } + return ret; +} + +// TODO: MessageToJsonString is a newly added function in protobuf +// and is not yet available in the android tree. Replace this function with +// https://developers.google.com/protocol-buffers/docs/reference/cpp/google.protobuf.util.json_util#MessageToJsonString.details +// when the android tree gets updated. +// +// The new MessageToJsonString also allows preserving proto field names. However, +// the function here can't. Hence, a field name "foo_bar" without json_name option +// will be "fooBar" in the final output. Additional checks are needed to ensure +// that doesn't happen. +ErrorOr<std::string> MessageToJsonString(const google::protobuf::Message& message); + +} // namespace jsonpb +} // namespace android diff --git a/lib/libjsonpb/parse/jsonpb.cpp b/lib/libjsonpb/parse/jsonpb.cpp new file mode 100755 index 0000000..d7feb67 --- /dev/null +++ b/lib/libjsonpb/parse/jsonpb.cpp @@ -0,0 +1,74 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <jsonpb/jsonpb.h> + +#include <android-base/logging.h> +#include <google/protobuf/descriptor.h> +#include <google/protobuf/message.h> +#include <google/protobuf/util/json_util.h> +#include <google/protobuf/util/type_resolver_util.h> + +namespace android { +namespace jsonpb { + +using google::protobuf::DescriptorPool; +using google::protobuf::Message; +using google::protobuf::util::NewTypeResolverForDescriptorPool; +using google::protobuf::util::TypeResolver; + +static constexpr char kTypeUrlPrefix[] = "type.googleapis.com"; + +std::string GetTypeUrl(const Message& message) { + return std::string(kTypeUrlPrefix) + "/" + message.GetDescriptor()->full_name(); +} + +ErrorOr<std::string> MessageToJsonString(const Message& message) { + std::unique_ptr<TypeResolver> resolver( + NewTypeResolverForDescriptorPool(kTypeUrlPrefix, DescriptorPool::generated_pool())); + + google::protobuf::util::JsonOptions options; + options.add_whitespace = true; + + std::string json; + auto status = BinaryToJsonString(resolver.get(), GetTypeUrl(message), + message.SerializeAsString(), &json, options); + + if (!status.ok()) { + return MakeError<std::string>(status.error_message().as_string()); + } + return ErrorOr<std::string>(std::move(json)); +} + +namespace internal { +ErrorOr<std::monostate> JsonStringToMessage(const std::string& content, Message* message) { + std::unique_ptr<TypeResolver> resolver( + NewTypeResolverForDescriptorPool(kTypeUrlPrefix, DescriptorPool::generated_pool())); + + std::string binary; + auto status = JsonToBinaryString(resolver.get(), GetTypeUrl(*message), content, &binary); + if (!status.ok()) { + return MakeError<std::monostate>(status.error_message().as_string()); + } + if (!message->ParseFromString(binary)) { + return MakeError<std::monostate>("Fail to parse."); + } + return ErrorOr<std::monostate>(); +} +} // namespace internal + +} // namespace jsonpb +} // namespace android diff --git a/lib/libjsonpb/verify/Android.bp b/lib/libjsonpb/verify/Android.bp new file mode 100755 index 0000000..b32b9b4 --- /dev/null +++ b/lib/libjsonpb/verify/Android.bp @@ -0,0 +1,70 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This static library defines parameterized tests that enforce additional restrictions when +// using Protobuf as schema for JSON files. The reason is that the JSON parser that +// libprotobuf-cpp-full provides is relatively relaxed. +cc_library_static { + name: "libjsonpbverify", + host_supported: true, + srcs: [ + "verify.cpp", + ], + shared_libs: [ + "libbase", + "libprotobuf-cpp-full", + "libjsoncpp", + ], + static_libs: [ + "libjsonpbparse", + ], + export_static_lib_headers: [ + "libjsonpbparse", + ], + export_include_dirs: [ + "include", + ], + cflags: [ + "-Wall", + "-Werror", + "-Wno-unused-parameter", + ], +} + +cc_test_host { + name: "libjsonpbverify_test", + srcs: [ + "test.cpp", + "test.proto", + ], + static_libs: [ + "libbase", + "liblog", + "libgmock", + "libjsoncpp", + "libjsonpbparse", + "libjsonpbverify", + ], + shared_libs: [ + "libprotobuf-cpp-full", + ], + cflags: [ + "-Wall", + "-Werror", + "-Wno-unused-parameter", + ], + test_suites: [ + "general-tests", + ], +} diff --git a/lib/libjsonpb/verify/include/jsonpb/json_schema_test.h b/lib/libjsonpb/verify/include/jsonpb/json_schema_test.h new file mode 100755 index 0000000..3db1931 --- /dev/null +++ b/lib/libjsonpb/verify/include/jsonpb/json_schema_test.h @@ -0,0 +1,121 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +#pragma once + +#include <unistd.h> + +#include <memory> +#include <string> + +#include <android-base/file.h> +#include <android-base/strings.h> +#include <gtest/gtest.h> +#include <json/reader.h> +#include <json/writer.h> +#include <jsonpb/jsonpb.h> +#include <jsonpb/verify.h> + +// JsonSchemaTest test that a given JSON file conforms to a given schema. +// This includes: +// - libprotobuf can parse the given JSON file using the given Prototype class +// - Additional checks on field names of the JSON file, and types of values. + +namespace android { +namespace jsonpb { + +class JsonSchemaTestConfig { + public: + virtual ~JsonSchemaTestConfig() = default; + virtual std::unique_ptr<google::protobuf::Message> CreateMessage() const = 0; + virtual std::string file_path() const = 0; + /** + * If it returns true, tests are skipped when the file is not found. + */ + virtual bool optional() const { + return false; + } +}; +using JsonSchemaTestConfigFactory = + std::function<std::unique_ptr<JsonSchemaTestConfig>()>; + +template <typename T> +class AbstractJsonSchemaTestConfig : public JsonSchemaTestConfig { + public: + AbstractJsonSchemaTestConfig(const std::string& path) : file_path_(path){}; + std::unique_ptr<google::protobuf::Message> CreateMessage() const override { + return std::make_unique<T>(); + } + std::string file_path() const override { return file_path_; } + + private: + std::string file_path_; +}; + +template <typename T> +JsonSchemaTestConfigFactory MakeTestParam(const std::string& path) { + return [path]() { + return std::make_unique<AbstractJsonSchemaTestConfig<T>>(path); + }; +} + +class JsonSchemaTest + : public ::testing::TestWithParam<JsonSchemaTestConfigFactory> { + public: + void SetUp() override { + auto&& config = + ::testing::TestWithParam<JsonSchemaTestConfigFactory>::GetParam()(); + file_path_ = config->file_path(); + + if (access(file_path_.c_str(), F_OK) == -1) { + ASSERT_EQ(ENOENT, errno) << "File '" << file_path_ << "' is not accessible: " + << strerror(errno); + ASSERT_TRUE(config->optional()) << "Missing mandatory file " << file_path_; + GTEST_SKIP(); + } + ASSERT_TRUE(android::base::ReadFileToString(file_path_, &json_)); + ASSERT_FALSE(json_.empty()) << "File '" << file_path_ << "' exists but is empty"; + + object_ = config->CreateMessage(); + auto res = internal::JsonStringToMessage(json_, object_.get()); + ASSERT_TRUE(res.ok()) << "Invalid format of file " << file_path_ + << ": " << res.error(); + } + google::protobuf::Message* message() const { + return object_.get(); + } + std::string file_path_; + std::string json_; + std::unique_ptr<google::protobuf::Message> object_; +}; + +// Test that the JSON file has no fields unknown by the schema. See +// AllFieldsAreKnown() for more details. +TEST_P(JsonSchemaTest, NoUnknownFields) { + std::string error; + EXPECT_TRUE(AllFieldsAreKnown(*object_, json_, &error)) + << "File: " << file_path_ << ": " << error; +} + +TEST_P(JsonSchemaTest, EqReformattedJson) { + std::string error; + EXPECT_TRUE(EqReformattedJson(json_, object_.get(), &error)) + << "File: " << file_path_ << ": " << error; +} + +} // namespace jsonpb +} // namespace android diff --git a/lib/libjsonpb/verify/include/jsonpb/verify.h b/lib/libjsonpb/verify/include/jsonpb/verify.h new file mode 100755 index 0000000..c05b13d --- /dev/null +++ b/lib/libjsonpb/verify/include/jsonpb/verify.h @@ -0,0 +1,93 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +#pragma once + +#include <sstream> +#include <string> +#include <vector> + +#include <google/protobuf/message.h> +#include <json/reader.h> +#include <json/value.h> +#include <jsonpb/jsonpb.h> + +namespace android { +namespace jsonpb { + +// Ensure that the JSON file has no unknown fields that is not defined in proto. +// Because we want forwards compatibility, the parser of JSON files must ignore +// unknown fields. This is achievable with libprotobuf version > 3.0-beta. +// - <= 3.0-beta: we have to check unknown fields manually, and parser cannot +// use libprotobuf +// to parse JSON files. +// - < 3.5: libprotobuf discards all unknown fields. We can still check unknown +// fields manually, but +// an easier way to check is `json == FormatJson(json)` (schematically) +// - >= 3.5: Unknown fields are preserved, so FormatJson() may contain these +// unknown fields. We can +// still check fields manually, or use reflection mechanism. +// +// For example, if a new field "foo" is added to cgroups.json but not to +// cgroups.proto, libprocessgroup could technically read the value of "foo" by +// using other libraries that parse JSON strings, effectively working around the +// schema. +// +// This test also ensures that the parser does not use alternative key names. +// For example, if the proto file states: message Foo { string foo_bar = 1; +// string bar_baz = 2 [json_name = "BarBaz"]; } Then the parser accepts +// "foo_bar" "fooBar", "bar_baz", "BarBaz" as valid key names. Here, we enforce +// that the JSON file must use "foo_bar" and "BarBaz". +// +// Requiring this avoids surprises like: +// message Foo { string FooBar = 1; } +// { "fooBar" : "s" } +// conforms with the schema, because libprotobuf accept "fooBar" as a valid key. +// The correct schema should be: +// message Foo { string foo_bar = 1 [json_name="fooBar"]; } +// +// Params: +// path: path to navigate inside JSON tree. For example, {"foo", "bar"} for +// the value "string" in +// {"foo": {"bar" : "string"}} +bool AllFieldsAreKnown(const google::protobuf::Message& message, + const std::string& json, std::string* error); + +// Format the given JSON string according to Prototype T. This will serialize +// the JSON string to a Prototype message, then re-print the message as JSON. By +// reformatting the JSON string, we effectively enforces that the JSON source +// file uses conventions of Protobuf's JSON writer; e.g. 64-bit integers / +// special floating point numbers (inf, NaN, etc.) in strings, enum values in +// names, etc. +// +// Params: +// scratch_space: The scratch space to use to store the Protobuf message. It +// must be a pointer +// to the schema that the JSON string conforms to. +bool EqReformattedJson(const std::string& json, + google::protobuf::Message* scratch_space, + std::string* error); + +namespace internal { +// See EqReformattedJson(). +ErrorOr<std::string> FormatJson(const std::string& json, + google::protobuf::Message* scratch_space); + +} // namespace internal + +} // namespace jsonpb +} // namespace android diff --git a/lib/libjsonpb/verify/test.cpp b/lib/libjsonpb/verify/test.cpp new file mode 100755 index 0000000..cb98f47 --- /dev/null +++ b/lib/libjsonpb/verify/test.cpp @@ -0,0 +1,302 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +#include <limits> + +#include <sstream> + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include <json/writer.h> +#include <jsonpb/jsonpb.h> +#include <jsonpb/verify.h> + +#include "test.pb.h" + +using ::android::jsonpb::internal::FormatJson; +using ::testing::ElementsAre; +using ::testing::HasSubstr; + +namespace android { +namespace jsonpb { + +// Unit tests for libjsonpbverify. + +class LibJsonpbVerifyTest : public ::testing::Test {}; + +class JsonKeyTest : public LibJsonpbVerifyTest { + public: + template <typename T> + std::string GetFieldJsonName(const std::string& field_name) { + return T{}.GetDescriptor()->FindFieldByName(field_name)->json_name(); + } + + template <typename T> + void TestParseOkWithUnknownKey(const std::string& field_name, + const std::string& json_key) { + std::string json = "{\"" + json_key + "\": \"test\"}"; + auto object = JsonStringToMessage<T>(json); + ASSERT_TRUE(object.ok()) << object.error(); + EXPECT_EQ( + "test", + object->GetReflection()->GetString( + *object, object->GetDescriptor()->FindFieldByName(field_name))); + std::string error; + ASSERT_FALSE(AllFieldsAreKnown(*object, json, &error)) + << "AllFieldsAreKnown should return false"; + EXPECT_THAT(error, HasSubstr("unknown keys")); + EXPECT_THAT(error, HasSubstr(json_key)); + } +}; + +TEST_F(JsonKeyTest, WithJsonNameOk) { + std::string json = + "{\n" + " \"FOOBAR\": \"foo_bar\",\n" + " \"BarBaz\": \"barBaz\",\n" + " \"baz_qux\": \"BazQux\",\n" + " \"quxQuux\": \"QUX_QUUX\"\n" + "\n}"; + auto object = JsonStringToMessage<WithJsonName>(json); + ASSERT_TRUE(object.ok()) << object.error(); + + EXPECT_EQ("foo_bar", object->foo_bar()); + EXPECT_EQ("barBaz", object->barbaz()); + EXPECT_EQ("BazQux", object->bazqux()); + EXPECT_EQ("QUX_QUUX", object->qux_quux()); + + std::string error; + EXPECT_TRUE(AllFieldsAreKnown(*object, json, &error)) << error; +} + +// If Prototype field name as keys while json_name is present, AllFieldsAreKnown +// should return false. +TEST_F(JsonKeyTest, WithJsonNameFooBar) { + TestParseOkWithUnknownKey<WithJsonName>("foo_bar", "foo_bar"); +} + +TEST_F(JsonKeyTest, WithJsonNameBarBaz) { + TestParseOkWithUnknownKey<WithJsonName>("barBaz", "barBaz"); +} + +TEST_F(JsonKeyTest, WithJsonNameBazQux) { + TestParseOkWithUnknownKey<WithJsonName>("BazQux", "BazQux"); +} + +TEST_F(JsonKeyTest, WithJsonNameQuxQuux) { + TestParseOkWithUnknownKey<WithJsonName>("QUX_QUUX", "QUX_QUUX"); +} + +// JSON field name matches Proto field name +TEST_F(JsonKeyTest, NoJsonNameOk) { + std::string json = + "{\n" + " \"foo_bar\": \"foo_bar\",\n" + " \"barBaz\": \"barBaz\",\n" + " \"BazQux\": \"BazQux\",\n" + " \"QUX_QUUX\": \"QUX_QUUX\"\n" + "\n}"; + auto object = JsonStringToMessage<NoJsonName>(json); + ASSERT_TRUE(object.ok()) << object.error(); + + EXPECT_EQ("foo_bar", object->foo_bar()); + EXPECT_EQ("barBaz", object->barbaz()); + EXPECT_EQ("BazQux", object->bazqux()); + EXPECT_EQ("QUX_QUUX", object->qux_quux()); + + std::string error; + EXPECT_TRUE(AllFieldsAreKnown(*object, json, &error)) << error; +} + +// JSON field name is lower/UpperCamelCase of Proto field name; +// AllFieldsAreKnown should return false. Although the lower/UpperCamelCase name +// is a valid key accepted by Protobuf's JSON parser, we explicitly disallow the +// behavior. +TEST_F(JsonKeyTest, NoJsonNameFooBar) { + EXPECT_EQ("fooBar", GetFieldJsonName<NoJsonName>("foo_bar")); + TestParseOkWithUnknownKey<NoJsonName>("foo_bar", "fooBar"); +} + +TEST_F(JsonKeyTest, NoJsonNameBarBaz) { + EXPECT_EQ("barBaz", GetFieldJsonName<NoJsonName>("barBaz")); + // No test for barBaz because its JSON name is the same as field_name +} + +TEST_F(JsonKeyTest, NoJsonNameBazQux) { + EXPECT_EQ("BazQux", GetFieldJsonName<NoJsonName>("BazQux")); + // No test for BazQux because its JSON name is the same as field_name +} + +TEST_F(JsonKeyTest, NoJsonNameQuxQuux) { + EXPECT_EQ("QUXQUUX", GetFieldJsonName<NoJsonName>("QUX_QUUX")); + TestParseOkWithUnknownKey<NoJsonName>("QUX_QUUX", "QUXQUUX"); +} + +class EmbeddedJsonKeyTest : public LibJsonpbVerifyTest { + public: + ErrorOr<Parent> TestEmbeddedError(const std::string& json, + const std::string& unknown_key) { + auto object = JsonStringToMessage<Parent>(json); + if (!object.ok()) return object; + std::string error; + EXPECT_FALSE(AllFieldsAreKnown(*object, json, &error)) + << "AllFieldsAreKnown should return false"; + EXPECT_THAT(error, HasSubstr("unknown keys")); + EXPECT_THAT(error, HasSubstr(unknown_key)); + return object; + } +}; + +TEST_F(EmbeddedJsonKeyTest, Ok) { + std::string json = + "{" + " \"with_json_name\": {\"FOOBAR\": \"foo_bar\"},\n" + " \"repeated_with_json_name\": [{\"BarBaz\": \"barBaz\"}],\n" + " \"no_json_name\": {\"BazQux\": \"BazQux\"},\n" + " \"repeated_no_json_name\": [{\"QUX_QUUX\": \"QUX_QUUX\"}]\n" + "}"; + auto object = JsonStringToMessage<Parent>(json); + ASSERT_TRUE(object.ok()) << object.error(); + + EXPECT_EQ("foo_bar", object->with_json_name().foo_bar()); + ASSERT_EQ(1u, object->repeated_with_json_name().size()); + EXPECT_EQ("barBaz", object->repeated_with_json_name().begin()->barbaz()); + EXPECT_EQ("BazQux", object->no_json_name().bazqux()); + ASSERT_EQ(1u, object->repeated_no_json_name().size()); + EXPECT_EQ("QUX_QUUX", object->repeated_no_json_name().begin()->qux_quux()); + + std::string error; + EXPECT_TRUE(AllFieldsAreKnown(*object, json, &error)) << error; +} + +TEST_F(EmbeddedJsonKeyTest, FooBar) { + auto object = TestEmbeddedError( + "{\"with_json_name\": {\"foo_bar\": \"test\"}}", "foo_bar"); + ASSERT_TRUE(object.ok()) << object.error(); + EXPECT_EQ("test", object->with_json_name().foo_bar()); +} + +TEST_F(EmbeddedJsonKeyTest, BarBaz) { + auto object = TestEmbeddedError( + "{\"repeated_with_json_name\": [{\"barBaz\": \"test\"}]}", "barBaz"); + ASSERT_TRUE(object.ok()) << object.error(); + ASSERT_EQ(1u, object->repeated_with_json_name().size()); + EXPECT_EQ("test", object->repeated_with_json_name().begin()->barbaz()); +} + +TEST_F(EmbeddedJsonKeyTest, NoJsonName) { + auto object = TestEmbeddedError( + "{\"no_json_name\": {\"QUXQUUX\": \"test\"}}", "QUXQUUX"); + ASSERT_TRUE(object.ok()) << object.error(); + EXPECT_EQ("test", object->no_json_name().qux_quux()); +} + +TEST_F(EmbeddedJsonKeyTest, QuxQuux) { + auto object = TestEmbeddedError( + "{\"repeated_no_json_name\": [{\"QUXQUUX\": \"test\"}]}", "QUXQUUX"); + ASSERT_TRUE(object.ok()) << object.error(); + ASSERT_EQ(1u, object->repeated_no_json_name().size()); + EXPECT_EQ("test", object->repeated_no_json_name().begin()->qux_quux()); +} + +class ScalarTest : public LibJsonpbVerifyTest { + public: + ::testing::AssertionResult IsJsonEq(const std::string& l, + const std::string& r) { + Json::Reader reader; + Json::Value lvalue; + if (!reader.parse(l, lvalue)) + return ::testing::AssertionFailure() + << reader.getFormattedErrorMessages(); + Json::Value rvalue; + if (!reader.parse(r, rvalue)) + return ::testing::AssertionFailure() + << reader.getFormattedErrorMessages(); + Json::StyledWriter writer; + return lvalue == rvalue + ? (::testing::AssertionSuccess() << "Both are \n" + << writer.write(lvalue)) + : (::testing::AssertionFailure() + << writer.write(lvalue) << "\n does not equal \n" + << writer.write(rvalue)); + } + + bool EqReformattedJson(const std::string& json, std::string* error) { + return android::jsonpb::EqReformattedJson(json, &scalar_, error); + } + + Scalar scalar_; + std::string error_; +}; + +TEST_F(ScalarTest, Ok) { + std::string json = + "{\n" + " \"i32\": 1,\n" + " \"si32\": 1,\n" + " \"i64\": \"1\",\n" + " \"si64\": \"1\",\n" + " \"f\": 1.5,\n" + " \"d\": 1.5,\n" + " \"e\": \"FOO\"\n" + "}"; + auto formatted = FormatJson(json, &scalar_); + ASSERT_TRUE(formatted.ok()) << formatted.error(); + EXPECT_TRUE(IsJsonEq(json, *formatted)); + + EXPECT_TRUE(EqReformattedJson(json, &error_)) << error_; +} + +using ScalarTestErrorParam = std::tuple<const char*, const char*>; +class ScalarTestError + : public ScalarTest, + public ::testing::WithParamInterface<ScalarTestErrorParam> {}; + +TEST_P(ScalarTestError, Test) { + std::string json; + std::string message; + std::tie(json, message) = GetParam(); + auto formatted = FormatJson(json, &scalar_); + ASSERT_TRUE(formatted.ok()) << formatted.error(); + EXPECT_FALSE(IsJsonEq(json, *formatted)) << message; + EXPECT_FALSE(EqReformattedJson(json, &error_)) + << "EqReformattedJson should return false"; +} + +static const std::vector<ScalarTestErrorParam> gScalarTestErrorParams = { + {"{\"i32\": \"1\"}", "Should not allow int32 values to be quoted"}, + {"{\"si32\": \"1\"}", "Should not allow sint32 values to be quoted"}, + {"{\"i64\": 1}", "Should require int64 values to be quoted"}, + {"{\"si64\": 1}", "Should require sint64 values to be quoted"}, + {"{\"f\": \"1.5\"}", "Should not allow float values to be quoted"}, + {"{\"d\": \"1.5\"}", "Should not allow double values to be quoted"}, + {"{\"e\": 1}", "Should not allow integers for enums"}, +}; + +INSTANTIATE_TEST_SUITE_P(Jsonpb, ScalarTestError, + ::testing::ValuesIn(gScalarTestErrorParams)); + +int main(int argc, char** argv) { + using ::testing::AddGlobalTestEnvironment; + using ::testing::InitGoogleTest; + + InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +} // namespace jsonpb +} // namespace android diff --git a/lib/libjsonpb/verify/test.proto b/lib/libjsonpb/verify/test.proto new file mode 100755 index 0000000..29ec8b1 --- /dev/null +++ b/lib/libjsonpb/verify/test.proto @@ -0,0 +1,65 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +package android.jsonpb; + +// Note: this file explicitly uses names that does NOT follow the Protobuf Style +// Guide for testing purposes. When writing a .proto file as a JSON schema, you +// should: +// - Follow the Protobuf Style Guide for field names / enum value names +// - If the JSON file is going to have field names that does not conform to the +// Protobuf Style Guide (a.k.a lower_snake_case), use json_name option to +// indicate an alternative name. +// - If the JSON file is going to have enum value names that does not conform to +// the Protobuf Style Guide (a.k.a CAPITALIZED_SNAKE_CASE), use strings. + +message WithJsonName { + string foo_bar = 1 [json_name = "FOOBAR"]; + string barBaz = 2 [json_name = "BarBaz"]; + string BazQux = 3 [json_name = "baz_qux"]; + string QUX_QUUX = 4 [json_name = "quxQuux"]; +} + +message NoJsonName { + string foo_bar = 1; + string barBaz = 2; + string BazQux = 3; + string QUX_QUUX = 4; +} + +message Parent { + repeated WithJsonName repeated_with_json_name = 1; + WithJsonName with_json_name = 2; + repeated NoJsonName repeated_no_json_name = 3; + NoJsonName no_json_name = 4; +} + +message Scalar { + int32 i32 = 1; + sint32 si32 = 2; + int64 i64 = 3; + sint64 si64 = 4; + float f = 5; + double d = 6; + + enum Enum { + DEFAULT = 0; + FOO = 1; + } + Enum e = 7; +} diff --git a/lib/libjsonpb/verify/verify.cpp b/lib/libjsonpb/verify/verify.cpp new file mode 100755 index 0000000..c411de8 --- /dev/null +++ b/lib/libjsonpb/verify/verify.cpp @@ -0,0 +1,221 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +#include <jsonpb/verify.h> + +#include <iostream> +#include <memory> +#include <sstream> +#include <string> + +#include <android-base/strings.h> +#include <google/protobuf/descriptor.h> +#include <google/protobuf/descriptor.pb.h> +#include <google/protobuf/message.h> +#include <google/protobuf/reflection.h> +#include <json/reader.h> +#include <json/writer.h> +#include <jsonpb/jsonpb.h> + +namespace android { +namespace jsonpb { + +using google::protobuf::FieldDescriptor; +using google::protobuf::FieldDescriptorProto; +using google::protobuf::Message; + +// Return json_name of the field. If it is not set, return the name of the +// field. +const std::string& GetJsonName(const FieldDescriptor& field_descriptor) { + // The current version of libprotobuf does not define + // FieldDescriptor::has_json_name() yet. Use a workaround. + // TODO: use field_descriptor.has_json_name() when libprotobuf version is + // bumped. + FieldDescriptorProto proto; + field_descriptor.CopyTo(&proto); + return proto.has_json_name() ? field_descriptor.json_name() + : field_descriptor.name(); +} + +bool AllFieldsAreKnown(const Message& message, const Json::Value& json, + std::vector<std::string>* path, + std::stringstream* error) { + if (!json.isObject()) { + *error << base::Join(*path, ".") << ": Not a JSON object\n"; + return false; + } + auto&& descriptor = message.GetDescriptor(); + + auto json_members = json.getMemberNames(); + std::set<std::string> json_keys{json_members.begin(), json_members.end()}; + + std::set<std::string> known_keys; + for (int i = 0; i < descriptor->field_count(); ++i) { + known_keys.insert(GetJsonName(*descriptor->field(i))); + } + + std::set<std::string> unknown_keys; + std::set_difference(json_keys.begin(), json_keys.end(), known_keys.begin(), + known_keys.end(), + std::inserter(unknown_keys, unknown_keys.begin())); + + if (!unknown_keys.empty()) { + *error << base::Join(*path, ".") << ": contains unknown keys: [" + << base::Join(unknown_keys, ", ") + << "]. Keys must be a known field name of " + << descriptor->full_name() << "(or its json_name option if set): [" + << base::Join(known_keys, ", ") << "]\n"; + return false; + } + + bool success = true; + + // Check message fields. + auto&& reflection = message.GetReflection(); + std::vector<const FieldDescriptor*> set_field_descriptors; + reflection->ListFields(message, &set_field_descriptors); + for (auto&& field_descriptor : set_field_descriptors) { + if (field_descriptor->cpp_type() != + FieldDescriptor::CppType::CPPTYPE_MESSAGE) { + continue; + } + if (field_descriptor->is_map()) { + continue; + } + + const std::string& json_name = GetJsonName(*field_descriptor); + const Json::Value& json_value = json[json_name]; + + if (field_descriptor->is_repeated()) { + auto&& fields = + reflection->GetRepeatedFieldRef<Message>(message, field_descriptor); + + if (json_value.type() != Json::ValueType::arrayValue) { + *error << base::Join(*path, ".") + << ": not a JSON list. This should not happen.\n"; + success = false; + continue; + } + + if (json_value.size() != static_cast<size_t>(fields.size())) { + *error << base::Join(*path, ".") << ": JSON list has size " + << json_value.size() << " but message has size " << fields.size() + << ". This should not happen.\n"; + success = false; + continue; + } + + std::unique_ptr<Message> scratch_space(fields.NewMessage()); + for (int i = 0; i < fields.size(); ++i) { + path->push_back(json_name + "[" + std::to_string(i) + "]"); + auto res = AllFieldsAreKnown(fields.Get(i, scratch_space.get()), + json_value[i], path, error); + path->pop_back(); + if (!res) { + success = false; + } + } + } else { + auto&& field = reflection->GetMessage(message, field_descriptor); + path->push_back(json_name); + auto res = AllFieldsAreKnown(field, json_value, path, error); + path->pop_back(); + if (!res) { + success = false; + } + } + } + return success; +} + +bool AllFieldsAreKnown(const google::protobuf::Message& message, + const std::string& json, std::string* error) { + Json::Reader reader; + Json::Value value; + if (!reader.parse(json, value)) { + *error = reader.getFormattedErrorMessages(); + return false; + } + + std::stringstream errorss; + std::vector<std::string> json_tree_path{"<root>"}; + if (!AllFieldsAreKnown(message, value, &json_tree_path, &errorss)) { + *error = errorss.str(); + return false; + } + return true; +} + +bool EqReformattedJson(const std::string& json, + google::protobuf::Message* scratch_space, + std::string* error) { + Json::Reader reader; + Json::Value old_json; + if (!reader.parse(json, old_json)) { + *error = reader.getFormattedErrorMessages(); + return false; + } + + auto new_json_string = internal::FormatJson(json, scratch_space); + if (!new_json_string.ok()) { + *error = new_json_string.error(); + return false; + } + Json::Value new_json; + if (!reader.parse(*new_json_string, new_json)) { + *error = reader.getFormattedErrorMessages(); + return false; + } + + if (old_json != new_json) { + std::stringstream ss; + ss << "Formatted JSON tree does not match source. Possible reasons " + "include: \n" + "- JSON Integers (without quotes) are matched against 64-bit " + "integers in Prototype\n" + " (Reformatted integers will now have quotes.) Quote these integers " + "in source\n" + " JSON or use 32-bit integers instead.\n" + "- Enum values are stored as integers in source JSON file. Use enum " + "value name \n" + " string instead, or change schema field to string / integers.\n" + "- JSON keys are re-formatted to be lowerCamelCase. To fix, define " + "json_name " + "option\n" + " for appropriate fields.\n" + "\n" + "Reformatted JSON is printed below.\n" + << Json::StyledWriter().write(new_json); + *error = ss.str(); + return false; + } + return true; +} + +namespace internal { +ErrorOr<std::string> FormatJson(const std::string& json, + google::protobuf::Message* scratch_space) { + auto res = internal::JsonStringToMessage(json, scratch_space); + if (!res.ok()) { + return MakeError<std::string>(res.error()); + } + return MessageToJsonString(*scratch_space); +} +} // namespace internal + +} // namespace jsonpb +} // namespace android |