diff options
author | Kyungwook Tak <k.tak@samsung.com> | 2016-08-29 10:14:52 +0900 |
---|---|---|
committer | Kyungwook Tak <k.tak@samsung.com> | 2016-08-29 10:16:56 +0900 |
commit | b3db23b4212b0d025f44205b22c7d5e8cfbb06b2 (patch) | |
tree | 03a2ba1c07661b473bc8d2755754e99f7167033e | |
parent | b432a168e7c882d8bf8d0c28bce345666ee9a511 (diff) | |
parent | 43202b711babc01aec018c1fb22265eea826c3bb (diff) | |
download | csr-framework-b3db23b4212b0d025f44205b22c7d5e8cfbb06b2.tar.gz csr-framework-b3db23b4212b0d025f44205b22c7d5e8cfbb06b2.tar.bz2 csr-framework-b3db23b4212b0d025f44205b22c7d5e8cfbb06b2.zip |
Release version 2.2.0submit/tizen/20160829.005907accepted/tizen/wearable/20160830.060740accepted/tizen/tv/20160830.060717accepted/tizen/mobile/20160830.060637accepted/tizen/ivi/20160830.060809accepted/tizen/common/20160829.135645
Main changes
Add db transaction
Client-server conversation arch for async scan
Upgrade db schema version for adding constraint
Change-Id: I15914dd748aa78bc1eccac790a7411c1d2ef1e13
Signed-off-by: Kyungwook Tak <k.tak@samsung.com>
44 files changed, 1366 insertions, 591 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index 34cc1e8..28a5d86 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -83,6 +83,7 @@ IF (PLATFORM_VERSION_3) SET(DECLARE_POPUP_SMACK_PROCESS_LABEL "") CONFIGURE_FILE(packaging/${SERVICE_NAME}.manifest.in ${SERVICE_NAME}.manifest @ONLY) CONFIGURE_FILE(packaging/${SERVICE_NAME}-test.manifest.in ${SERVICE_NAME}-test.manifest @ONLY) + CONFIGURE_FILE(data/scripts/${SERVICE_NAME}-upgrade.sh.in data/scripts/${SERVICE_NAME}-upgrade.sh @ONLY) ELSE (PLATFORM_VERSION_3) SET(DECLARE_POPUP_USER User=app) SET(DECLARE_POPUP_GROUP Group=app) diff --git a/data/scripts/create_schema.sql b/data/scripts/create_schema.sql index f690928..eadf6bf 100644 --- a/data/scripts/create_schema.sql +++ b/data/scripts/create_schema.sql @@ -16,7 +16,7 @@ /* * @file create_schema.sql * @author Kyungwook Tak (k.tak@samsung.com) - * @version 1.0 + * @version 2.0 * @brief DB schema */ CREATE TABLE IF NOT EXISTS SCHEMA_INFO ( @@ -72,7 +72,8 @@ CREATE TABLE IF NOT EXISTS DETECTED_MALWARE_CLOUD ( severity INTEGER NOT NULL, detected_time INTEGER NOT NULL, - FOREIGN KEY(idx) REFERENCES NAMES(idx) ON DELETE CASCADE + FOREIGN KEY(idx) REFERENCES NAMES(idx) ON DELETE CASCADE, + UNIQUE(idx) ); CREATE TABLE IF NOT EXISTS PACKAGE_INFO ( diff --git a/data/scripts/csr-upgrade.sh.in b/data/scripts/csr-upgrade.sh.in new file mode 100644 index 0000000..66b30a3 --- /dev/null +++ b/data/scripts/csr-upgrade.sh.in @@ -0,0 +1,25 @@ +#!/bin/bash +# +# Copyright (c) 2016 Samsung Electronics Co., Ltd All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# @file csr-upgrade.sh.in +# @author Kyungwook Tak (k.tak@samsung.com) +# @brief Create RW data/directory for platform upgrade(2.4->3.0) scenario +# +PATH=/bin:/usr/bin:/sbin:/usr/sbin + +mkdir -p @RW_DBSPACE@ @ENGINE_RW_WORKING_DIR@ +chown @SERVICE_USER@:@SERVICE_GROUP@ @RW_DBSPACE@ @ENGINE_RW_WORKING_DIR@ +chsmack -t -a @SMACK_DOMAIN_NAME@ @RW_DBSPACE@ @ENGINE_RW_WORKING_DIR@ diff --git a/data/scripts/migrate_1.sql b/data/scripts/migrate_1.sql new file mode 100644 index 0000000..563daae --- /dev/null +++ b/data/scripts/migrate_1.sql @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2016 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License + */ +/* + * @file migrate_1.sql + * @author Kyungwook Tak (k.tak@samsung.com) + * @version 1.0 + * @brief DB migration from schema version 1 to 2 + */ + +-- isolate old data +DROP INDEX c_name_index; +DROP VIEW join_detecteds_cloud_by_name; +ALTER TABLE DETECTED_MALWARE_CLOUD RENAME TO OLD_DETECTED_MALWARE_CLOUD; + +-- create new structure +CREATE TABLE DETECTED_MALWARE_CLOUD ( + idx INTEGER NOT NULL, + pkg_id TEXT NOT NULL, + data_version TEXT NOT NULL, + malware_name TEXT NOT NULL, + detailed_url TEXT NOT NULL, + severity INTEGER NOT NULL, + detected_time INTEGER NOT NULL, + + FOREIGN KEY(idx) REFERENCES NAMES(idx) ON DELETE CASCADE, + UNIQUE(idx) +); + +CREATE VIEW [join_detecteds_cloud_by_name] AS + SELECT N.name, D.data_version, D.malware_name, D.detailed_url, D.severity, + D.detected_time, D.pkg_id, N.is_ignored + FROM NAMES AS N INNER JOIN DETECTED_MALWARE_CLOUD AS D ON N.idx = D.idx; +CREATE INDEX c_name_index ON DETECTED_MALWARE_CLOUD(idx); + +-- move data +INSERT INTO DETECTED_MALWARE_CLOUD(idx, pkg_id, data_version, malware_name, detailed_url, severity, detected_time) + SELECT idx, pkg_id, data_version, malware_name, detailed_url, severity, MAX(detected_time) FROM OLD_DETECTED_MALWARE_CLOUD GROUP BY idx; + +-- cleanup +DROP TABLE OLD_DETECTED_MALWARE_CLOUD; diff --git a/packaging/csr-framework.spec b/packaging/csr-framework.spec index af78217..284f818 100644 --- a/packaging/csr-framework.spec +++ b/packaging/csr-framework.spec @@ -22,7 +22,7 @@ Summary: A general purpose content screening and reputation solution Name: csr-framework -Version: 2.1.1 +Version: 2.2.0 Release: 0 Source: %{name}-%{version}.tar.gz License: Apache-2.0 and BSL-1.0 @@ -73,6 +73,7 @@ file contents and checking url to prevent malicious items. %global popup_service_env_file_path /run/tizen-system-env %global smack_domain_name System %global popup_unitdir %{_unitdir_user} +%global upgrade_script_dir %{ro_data_dir}/upgrade/scripts %else %global service_user system %global service_group system @@ -212,6 +213,11 @@ mkdir -p %{buildroot}%{rw_db_dir} mkdir -p %{buildroot}%{ro_db_dir} cp data/scripts/*.sql %{buildroot}%{ro_db_dir} +%if "%{?tizen_version}" == "3.0" +mkdir -p %{buildroot}%{upgrade_script_dir} +cp data/scripts/%{service_name}-upgrade.sh %{buildroot}%{upgrade_script_dir} +%endif + mkdir -p %{buildroot}%{engine_dir} mkdir -p %{buildroot}%{engine_rw_working_dir} @@ -294,6 +300,11 @@ chsmack -a "_" %{test_dir}/test_dir/dir1 %dir %{engine_dir} %dir %attr(775, %{service_user}, %{service_group}) %{engine_rw_working_dir} +# RW area platform upgrade script +%if "%{?tizen_version}" == "3.0" +%attr(755, -, -) %{upgrade_script_dir}/%{service_name}-upgrade.sh +%endif + %files -n lib%{name}-common %defattr(-,root,root,-) %manifest %{service_name}-common.manifest diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 788787a..a17ad25 100755 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -63,6 +63,7 @@ SET(${TARGET_CSR_SERVER}_SRCS framework/db/connection.cpp framework/db/statement.cpp framework/db/manager.cpp + framework/db/cache.cpp ${AC_BACKEND_SRCS} ) @@ -102,6 +103,7 @@ SET(${TARGET_CSR_CLIENT}_SRCS framework/client/canonicalize.cpp framework/client/content-screening.cpp framework/client/engine-manager.cpp + framework/client/eventfd.cpp framework/client/handle.cpp framework/client/handle-ext.cpp framework/client/utils.cpp diff --git a/src/framework/client/async-logic.cpp b/src/framework/client/async-logic.cpp index 0dec60a..43ca501 100644 --- a/src/framework/client/async-logic.cpp +++ b/src/framework/client/async-logic.cpp @@ -23,122 +23,114 @@ #include <cstdint> #include <utility> +#include <sys/epoll.h> #include "common/exception.h" #include "common/cs-detected.h" +#include "common/connection.h" +#include "common/async-protocol.h" #include "common/audit/logger.h" namespace Csr { namespace Client { AsyncLogic::AsyncLogic(HandleExt *handle, void *userdata) : - m_handle(handle), - m_ctx(new CsContext), - m_cb(handle->m_cb), - m_userdata(userdata) + m_handle(handle), m_userdata(userdata), m_dispatcherAsync(new Dispatcher(SockId::CS)) { - // disable ask user option for async request for now - copyKvp<int>(CsContext::Key::CoreUsage); - copyKvp<std::string>(CsContext::Key::PopupMessage); - copyKvp<bool>(CsContext::Key::ScanOnCloud); } -AsyncLogic::~AsyncLogic() +void AsyncLogic::stop(void) { - for (auto &resultPtr : this->m_results) - this->m_handle->add(std::move(resultPtr)); + INFO("async logic stop called! Let's send cancel signal to loop"); + this->m_dispatcherAsync->methodPing(CommandId::CANCEL_OPERATION); + this->m_cancelSignal.send(); } void AsyncLogic::scanDirs(const StrSet &dirs) { - for (const auto &dir : dirs) - this->scanDir(dir); + this->scanHelper(CommandId::SCAN_DIRS_ASYNC, dirs); } -void AsyncLogic::scanDir(const std::string &dir) +void AsyncLogic::scanFiles(const StrSet &files) { - auto startTime = ::time(nullptr); + this->scanHelper(CommandId::SCAN_FILES_ASYNC, files); +} - if (this->m_handle->isStopped()) - ThrowExcInfo(-999, "Async operation cancelled!"); +void AsyncLogic::scanHelper(const CommandId &id, const StrSet &s) +{ + this->m_dispatcherAsync->methodPing(id, *(this->m_handle->getContext()), s); - // Already scanned files are included in history. it'll be skipped later - // on server side by every single scan_file request. - auto retFiles = this->m_handle->dispatch<std::pair<int, std::shared_ptr<StrSet>>>( - CommandId::GET_SCANNABLE_FILES, dir); + auto fd = this->m_dispatcherAsync->getFd(); + auto cancelEventFd = this->m_cancelSignal.getFd(); - if (retFiles.first == -999) { - ThrowExcInfo(-999, "Async op cancelled!"); - } else if (retFiles.first != CSR_ERROR_NONE) { - ThrowExc(retFiles.first, "Error to get scannalbe files. " - "dir: " << dir << " ret: " << retFiles.first); - } + this->m_loop.addEventSource(cancelEventFd, EPOLLIN, + [&](uint32_t) { + this->m_cancelSignal.receive(); + ThrowExcInfo(ASYNC_EVENT_CANCEL, "Async event cancelled on fd: " << fd); + }); - if (retFiles.second == nullptr) { - INFO("No scannable file exist on dir: " << dir); - return; - } + this->m_loop.addEventSource(fd, EPOLLIN | EPOLLHUP | EPOLLRDHUP, + [&](uint32_t e) { + if (e & (EPOLLHUP | EPOLLRDHUP)) + ThrowExc(CSR_ERROR_SOCKET, "csr-server might be crashed. Finish async client loop"); -#ifdef TIZEN_DEBUG_ENABLE - DEBUG("scannable file list in dir[" << dir << - "], count[" << retFiles.second->size() << "]:"); - size_t count = 0; - for (const auto &file : *(retFiles.second)) - DEBUG(std::to_string(++count) << " : " << file); -#endif - - // Let's start scan files! - this->scanFiles(*(retFiles.second)); - - auto ts64 = static_cast<int64_t>(startTime); - - auto ret = this->m_handle->dispatch<int>(CommandId::SET_DIR_TIMESTAMP, dir, ts64); - if (ret != CSR_ERROR_NONE) - ERROR("Failed to set dir timestamp after scan dir[" << dir << "] with " - "ec[" << ret << "] This is server error and not affects to " - "client / scan result when it doesn't comes to delta scanning... " - "So just ignore this error on client side."); -} + // read event + auto event = this->m_dispatcherAsync->receiveEvent<int>(); -void AsyncLogic::scanFiles(const StrSet &fileSet) -{ - for (const auto &file : fileSet) { - if (this->m_handle->isStopped()) - ThrowExcInfo(-999, "Async op cancelled!"); - - auto ret = this->m_handle->dispatch<std::pair<int, CsDetected *>>( - CommandId::SCAN_FILE, this->m_ctx, file); - - // for auto memory deleting in case of exception - ResultPtr resultPtr(ret.second); - - // ignore all file-system related error in async operation. - if (ret.first == CSR_ERROR_FILE_DO_NOT_EXIST || - ret.first == CSR_ERROR_FILE_CHANGED || - ret.first == CSR_ERROR_FILE_SYSTEM) { - WARN("File system related error code returned when scan files async." - " Ignore all file-system related error in async operation because" - " scan file list has been provided by server." - " file: " << file << " ret: " << ret.first); - continue; + DEBUG("event received: " << event); + + switch (event) { + case ASYNC_EVENT_MALWARE_NONE: { + DEBUG("ASYNC_EVENT_MALWARE_NONE comes in!"); + auto targetName = this->m_dispatcherAsync->receiveEvent<std::string>(); + + if (targetName.empty()) { + ERROR("scanned event received but target name is empty"); + break; + } + + if (this->m_handle->m_cb.onScanned != nullptr) + this->m_handle->m_cb.onScanned(targetName.c_str(), this->m_userdata); + + break; } - if (ret.first != CSR_ERROR_NONE) - ThrowExc(ret.first, "Error on async scan. ret: " << ret.first << - " while scan file: " << file); + case ASYNC_EVENT_MALWARE_DETECTED: { + DEBUG("ASYNC_EVENT_MALWARE_DETECTED comes in!"); + auto malware = this->m_dispatcherAsync->receiveEvent<CsDetected *>(); + + if (malware == nullptr) { + ERROR("malware detected event received but handle is null"); + break; + } - if (ret.second) { - INFO("[Detected] file[" << file << "]"); - this->m_results.emplace_back(std::move(resultPtr)); + ResultPtr resultPtr(malware); - if (this->m_cb.onDetected != nullptr) - this->m_cb.onDetected(reinterpret_cast<csr_cs_malware_h>(ret.second), - this->m_userdata); - } else { - DEBUG("[Scanned] file[" << file << "]"); + if (this->m_handle->m_cb.onDetected != nullptr) { + this->m_handle->add(std::move(resultPtr)); + this->m_handle->m_cb.onDetected( + reinterpret_cast<csr_cs_malware_h>(malware), this->m_userdata); + } - if (this->m_cb.onScanned != nullptr) - this->m_cb.onScanned(file.c_str(), this->m_userdata); + break; + } + + default: + ThrowExcInfo(event, "Async event loop terminated by event: " << event); + } + }); + + try { + while (true) + this->m_loop.dispatch(-1); + } catch (const Exception &e) { + switch (e.error()) { + case ASYNC_EVENT_COMPLETE: + INFO("Async operation completed"); + break; + + default: + throw; } } } diff --git a/src/framework/client/async-logic.h b/src/framework/client/async-logic.h index a1294ec..7e82857 100644 --- a/src/framework/client/async-logic.h +++ b/src/framework/client/async-logic.h @@ -22,12 +22,13 @@ #pragma once #include <memory> -#include <atomic> #include "common/types.h" -#include "common/cs-context.h" -#include "client/callback.h" +#include "common/command-id.h" +#include "common/dispatcher.h" +#include "common/mainloop.h" #include "client/handle-ext.h" +#include "client/eventfd.h" namespace Csr { namespace Client { @@ -35,35 +36,22 @@ namespace Client { class AsyncLogic { public: AsyncLogic(HandleExt *handle, void *userdata); - virtual ~AsyncLogic(); + virtual ~AsyncLogic() = default; void scanFiles(const StrSet &files); - void scanDir(const std::string &dir); void scanDirs(const StrSet &dirs); void stop(void); private: - template<typename T> - void copyKvp(CsContext::Key); + void scanHelper(const CommandId &id, const StrSet &s); HandleExt *m_handle; // for registering results for auto-release - - ContextPtr m_ctx; - std::vector<ResultPtr> m_results; - - Callback m_cb; void *m_userdata; + Mainloop m_loop; + EventFd m_cancelSignal; + std::unique_ptr<Dispatcher> m_dispatcherAsync; }; -template<typename T> -void AsyncLogic::copyKvp(CsContext::Key key) -{ - T value; - - this->m_handle->getContext()->get(static_cast<int>(key), value); - this->m_ctx->set(static_cast<int>(key), value); -} - } } diff --git a/src/framework/client/canonicalize.cpp b/src/framework/client/canonicalize.cpp index 4846691..bb412dd 100644 --- a/src/framework/client/canonicalize.cpp +++ b/src/framework/client/canonicalize.cpp @@ -55,32 +55,5 @@ std::string getAbsolutePath(const std::string &path) return apath; } -void eraseSubdirectories(StrSet &dirset) -{ - if (dirset.size() < 2) - return; - - for (auto it = dirset.begin(); it != dirset.end(); ++it) { - auto itsub = it; - ++itsub; - while (true) { - if (itsub == dirset.end()) - break; - - auto itl = it->length(); - auto itsubl = itsub->length(); - - if (itl + 1 >= itsubl || // to short to be sub-directory - itsub->compare(0, itl, *it) != 0 || // prefix isn't matched - (*it != "/" && itsub->at(itl) != '/')) { // has '/' at the end of prefix - ++itsub; - continue; - } - - itsub = dirset.erase(itsub); - } - } -} - } } diff --git a/src/framework/client/canonicalize.h b/src/framework/client/canonicalize.h index 944b01d..16427e3 100644 --- a/src/framework/client/canonicalize.h +++ b/src/framework/client/canonicalize.h @@ -32,8 +32,5 @@ namespace Client { // based on linux function: realpath std::string getAbsolutePath(const std::string &path); -// input directory set should contains resolved path only -void eraseSubdirectories(StrSet &dirset); - } // namespace Client } // namespace Csr diff --git a/src/framework/client/content-screening.cpp b/src/framework/client/content-screening.cpp index 98f9a5c..17d518a 100644 --- a/src/framework/client/content-screening.cpp +++ b/src/framework/client/content-screening.cpp @@ -34,6 +34,7 @@ #include "common/cs-context.h" #include "common/cs-detected.h" #include "common/command-id.h" +#include "common/async-protocol.h" #include "common/audit/logger.h" using namespace Csr; @@ -228,10 +229,14 @@ int csr_cs_scan_data(csr_cs_context_h handle, const unsigned char *data, hExt->getContext(), RawBuffer(data, data + length)); - if (ret.second) - hExt->add(ResultPtr(ret.second)); + ResultPtr resultPtr(ret.second); - *malware = reinterpret_cast<csr_cs_malware_h>(ret.second); + if (ret.second && !ret.second->malwareName.empty()) { + hExt->add(std::move(resultPtr)); + *malware = reinterpret_cast<csr_cs_malware_h>(ret.second); + } else { + *malware = nullptr; + } return ret.first; @@ -255,10 +260,14 @@ int csr_cs_scan_file(csr_cs_context_h handle, const char *file_path, hExt->getContext(), Client::getAbsolutePath(file_path)); - if (ret.second) - hExt->add(ResultPtr(ret.second)); + ResultPtr resultPtr(ret.second); - *malware = reinterpret_cast<csr_cs_malware_h>(ret.second); + if (ret.second && !ret.second->malwareName.empty()) { + hExt->add(std::move(resultPtr)); + *malware = reinterpret_cast<csr_cs_malware_h>(ret.second); + } else { + *malware = nullptr; + } return ret.first; @@ -278,6 +287,7 @@ int csr_cs_set_file_scanned_cb(csr_cs_context_h handle, csr_cs_file_scanned_cb c auto hExt = reinterpret_cast<Client::HandleExt *>(handle); hExt->m_cb.onScanned = callback; + hExt->getContext()->set(static_cast<int>(CsContext::Key::ScannedCbRegistered), true); return CSR_ERROR_NONE; @@ -390,26 +400,14 @@ int csr_cs_scan_files_async(csr_cs_context_h handle, const char *file_paths[], auto task = std::make_shared<Task>([hExt, user_data, fileSet] { EXCEPTION_ASYNC_SAFE_START(hExt->m_cb, user_data) - if (hExt->isStopped()) - ThrowExcInfo(-999, "Async operation cancelled!"); - - auto ret = hExt->dispatch<std::pair<int, std::shared_ptr<StrSet>>>( - CommandId::CANONICALIZE_PATHS, *fileSet); - - if (ret.first != CSR_ERROR_NONE) - ThrowExc(ret.first, "Error on getting canonicalized paths in subthread. " - "ret: " << ret.first); - - std::shared_ptr<StrSet> canonicalizedFiles; + Client::AsyncLogic l(hExt, user_data); - if (ret.second == nullptr) - canonicalizedFiles = std::make_shared<StrSet>(); - else - canonicalizedFiles = std::move(ret.second); + hExt->setStopFunc([&l]() { l.stop(); }); - Client::AsyncLogic l(hExt, user_data); + if (hExt->isStopped()) + ThrowExcInfo(ASYNC_EVENT_CANCEL, "Async operation cancelled!"); - l.scanFiles(*canonicalizedFiles); + l.scanFiles(*fileSet); EXCEPTION_SAFE_END }); @@ -427,39 +425,9 @@ API int csr_cs_scan_dir_async(csr_cs_context_h handle, const char *dir_path, void *user_data) { - EXCEPTION_SAFE_START + const char *dir_paths[1] = { dir_path }; - if (handle == nullptr) - return CSR_ERROR_INVALID_HANDLE; - else if (dir_path == nullptr || dir_path[0] == '\0') - return CSR_ERROR_INVALID_PARAMETER; - - auto hExt = reinterpret_cast<Client::HandleExt *>(handle); - - if (hExt->isRunning()) { - ERROR("Async scanning already running with this handle."); - return CSR_ERROR_BUSY; - } - - auto dir = std::make_shared<std::string>(Client::getAbsolutePath(dir_path)); - - auto task = std::make_shared<Task>([hExt, user_data, dir] { - EXCEPTION_ASYNC_SAFE_START(hExt->m_cb, user_data) - - Client::AsyncLogic l(hExt, user_data); - - l.scanDir(*dir); - - EXCEPTION_SAFE_END - }); - - std::lock_guard<std::mutex> l(hExt->m_dispatchMutex); - - hExt->dispatchAsync(task); - - return CSR_ERROR_NONE; - - EXCEPTION_SAFE_END + return ::csr_cs_scan_dirs_async(handle, dir_paths, 1, user_data); } API @@ -492,28 +460,14 @@ int csr_cs_scan_dirs_async(csr_cs_context_h handle, const char *dir_paths[], auto task = std::make_shared<Task>([hExt, user_data, dirSet] { EXCEPTION_ASYNC_SAFE_START(hExt->m_cb, user_data) - if (hExt->isStopped()) - ThrowExcInfo(-999, "Async operation cancelled!"); - - auto ret = hExt->dispatch<std::pair<int, std::shared_ptr<StrSet>>>( - CommandId::CANONICALIZE_PATHS, *dirSet); - - if (ret.first != CSR_ERROR_NONE) - ThrowExc(ret.first, "Error on getting canonicalized paths in subthread. " - "ret: " << ret.first); - - std::shared_ptr<StrSet> canonicalizedDirs; - - if (ret.second == nullptr) - canonicalizedDirs = std::make_shared<StrSet>(); - else - canonicalizedDirs = std::move(ret.second); + Client::AsyncLogic l(hExt, user_data); - Client::eraseSubdirectories(*canonicalizedDirs); + hExt->setStopFunc([&l]() { l.stop(); }); - Client::AsyncLogic l(hExt, user_data); + if (hExt->isStopped()) + ThrowExcInfo(ASYNC_EVENT_CANCEL, "Async operation cancelled!"); - l.scanDirs(*canonicalizedDirs); + l.scanDirs(*dirSet); EXCEPTION_SAFE_END }); @@ -540,10 +494,6 @@ int csr_cs_cancel_scanning(csr_cs_context_h handle) if (!hExt->isRunning() || hExt->isStopped()) return CSR_ERROR_NO_TASK; - hExt->turnOnStopFlagOnly(); - - hExt->ping(CommandId::CANCEL_OPERATION); - hExt->stop(); return CSR_ERROR_NONE; @@ -742,10 +692,14 @@ int csr_cs_get_detected_malware(csr_cs_context_h handle, const char *file_path, auto ret = hExt->dispatch<std::pair<int, CsDetected *>>( CommandId::GET_DETECTED, Client::getAbsolutePath(file_path)); - if (ret.second) - hExt->add(ResultPtr(ret.second)); + ResultPtr resultPtr(ret.second); - *malware = reinterpret_cast<csr_cs_malware_h>(ret.second); + if (ret.second && !ret.second->malwareName.empty()) { + hExt->add(std::move(resultPtr)); + *malware = reinterpret_cast<csr_cs_malware_h>(ret.second); + } else { + *malware = nullptr; + } return ret.first; @@ -818,10 +772,14 @@ int csr_cs_get_ignored_malware(csr_cs_context_h handle, const char *file_path, auto ret = hExt->dispatch<std::pair<int, CsDetected *>>( CommandId::GET_IGNORED, Client::getAbsolutePath(file_path)); - if (ret.second) - hExt->add(ResultPtr(ret.second)); + ResultPtr resultPtr(ret.second); - *malware = reinterpret_cast<csr_cs_malware_h>(ret.second); + if (ret.second && !ret.second->malwareName.empty()) { + hExt->add(std::move(resultPtr)); + *malware = reinterpret_cast<csr_cs_malware_h>(ret.second); + } else { + *malware = nullptr; + } return ret.first; diff --git a/src/framework/client/eventfd.cpp b/src/framework/client/eventfd.cpp new file mode 100644 index 0000000..ec3d9ed --- /dev/null +++ b/src/framework/client/eventfd.cpp @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2016 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License + */ +/* + * @file eventfd.cpp + * @author Jaemin Ryu (jm77.ryu@samsung.com) + * @version 1.0 + * @brief + */ +#include "client/eventfd.h" + +#include <sys/types.h> +#include <unistd.h> +#include <cstdint> + +#include "common/exception.h" + +namespace Csr { +namespace Client { + +EventFd::EventFd(unsigned int initval, int flags) : + m_fd(::eventfd(initval, flags)) +{ + if (this->m_fd == -1) + ThrowExc(CSR_ERROR_SERVER, "Eventfd from constructor is failed!"); +} + +EventFd::~EventFd() +{ + if (this->m_fd != -1) + ::close(this->m_fd); +} + +void EventFd::send() +{ + const std::uint64_t val = 1; + if (::write(this->m_fd, &val, sizeof(val)) == -1) + ThrowExc(CSR_ERROR_SOCKET, "EventFd send to fd[" << this->m_fd << "] is failed!"); +} + +void EventFd::receive() +{ + std::uint64_t val; + if (::read(this->m_fd, &val, sizeof(val)) == -1) + ThrowExc(CSR_ERROR_SOCKET, "EventFd receive from fd[" << this->m_fd << "] is failed!"); +} + +} +} diff --git a/src/framework/client/eventfd.h b/src/framework/client/eventfd.h new file mode 100644 index 0000000..b6719d1 --- /dev/null +++ b/src/framework/client/eventfd.h @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2016 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License + */ +/* + * @file eventfd.h + * @author Jaemin Ryu (jm77.ryu@samsung.com) + * @version 1.0 + * @brief + */ +#pragma once + +#include <sys/eventfd.h> + +namespace Csr { +namespace Client { + +class EventFd { +public: + EventFd(unsigned int initval = 0, int flags = EFD_SEMAPHORE | EFD_CLOEXEC); + ~EventFd(); + + EventFd(const EventFd &) = delete; + EventFd &operator=(const EventFd &) = delete; + + void send(); + void receive(); + + inline int getFd() const noexcept + { + return this->m_fd; + } + +private: + int m_fd; +}; + +} +} diff --git a/src/framework/client/handle-ext.cpp b/src/framework/client/handle-ext.cpp index d0e6fda..66a9273 100644 --- a/src/framework/client/handle-ext.cpp +++ b/src/framework/client/handle-ext.cpp @@ -41,17 +41,23 @@ HandleExt::~HandleExt() this->m_worker.join(); } -void HandleExt::turnOnStopFlagOnly() +void HandleExt::setStopFunc(std::function<void()> &&func) { std::lock_guard<std::mutex> l(this->m_flagMutex); - this->m_stop = true; + this->m_stopFunc = std::move(func); } void HandleExt::stop() { DEBUG("Stop & join worker..."); - this->turnOnStopFlagOnly(); + { + std::lock_guard<std::mutex> l(this->m_flagMutex); + this->m_stop = true; + + if (this->m_stopFunc != nullptr) + this->m_stopFunc(); + } if (this->m_worker.joinable()) this->m_worker.join(); diff --git a/src/framework/client/handle-ext.h b/src/framework/client/handle-ext.h index d9789a0..31efdf9 100644 --- a/src/framework/client/handle-ext.h +++ b/src/framework/client/handle-ext.h @@ -37,7 +37,7 @@ public: virtual ~HandleExt(); void dispatchAsync(const std::shared_ptr<Task> &task); - void turnOnStopFlagOnly(void); + void setStopFunc(std::function<void()> &&func); void stop(void); bool isStopped(void) const; bool isRunning(void) const; @@ -52,6 +52,7 @@ private: bool m_stop; bool m_isRunning; std::thread m_worker; + std::function<void()> m_stopFunc; mutable std::mutex m_resultsMutex; mutable std::mutex m_flagMutex; }; diff --git a/src/framework/client/handle.h b/src/framework/client/handle.h index a398c0a..df2ab15 100644 --- a/src/framework/client/handle.h +++ b/src/framework/client/handle.h @@ -43,6 +43,9 @@ public: template<typename ...Args> void ping(Args &&...); + template<typename Type> + Type revent(void); + virtual void add(ResultPtr &&); virtual void add(ResultListPtr &&); @@ -71,5 +74,11 @@ void Handle::ping(Args &&...args) this->m_dispatcher->methodPing(std::forward<Args>(args)...); } +template<typename Type> +Type Handle::revent() +{ + return this->m_dispatcher->receiveEvent<Type>(); +} + } // namespace Client } // namespace Csr diff --git a/src/framework/client/utils.cpp b/src/framework/client/utils.cpp index da69690..98adece 100644 --- a/src/framework/client/utils.cpp +++ b/src/framework/client/utils.cpp @@ -25,6 +25,8 @@ #include "common/audit/logger.h" #include "common/exception.h" +#include "common/async-protocol.h" + #include <csr-error.h> __attribute__((constructor)) @@ -69,7 +71,7 @@ void exceptionGuardAsync(const Callback &callbacks, void *userdata, if (callbacks.onCompleted != nullptr) callbacks.onCompleted(userdata); } catch (const Exception &e) { - if (e.error() == -999) { + if (e.error() == ASYNC_EVENT_CANCEL) { INFO("Async operation cancel exception!"); if (callbacks.onCancelled != nullptr) callbacks.onCancelled(userdata); diff --git a/src/framework/common/async-protocol.h b/src/framework/common/async-protocol.h new file mode 100644 index 0000000..da74158 --- /dev/null +++ b/src/framework/common/async-protocol.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2016 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License + */ +/* + * @file async-protocol.h + * @author Kyungwook Tak (k.tak@samsung.com) + * @version 1.0 + * @brief Protocol for asynchronous scanning operations + */ +#pragma once + +namespace Csr { + +// should be positive values not to be conflicted with error code in csr-error.h +typedef enum { + ASYNC_EVENT_START = 0x10, // operation started + ASYNC_EVENT_COMPLETE = 0x20, // operation completed + ASYNC_EVENT_CANCEL = 0x30, // operation cancelled + ASYNC_EVENT_MALWARE_NONE = 0x40, // target scanned and no malware detected + ASYNC_EVENT_MALWARE_DETECTED = 0x50, // target scanned and malware detected +} async_op_protocol_e; + +} diff --git a/src/framework/common/command-id.h b/src/framework/common/command-id.h index c4b0936..497c2b7 100644 --- a/src/framework/common/command-id.h +++ b/src/framework/common/command-id.h @@ -33,9 +33,8 @@ enum class CommandId : int { GET_DETECTED_LIST = 0x1102, GET_IGNORED = 0x1103, GET_IGNORED_LIST = 0x1104, - GET_SCANNABLE_FILES = 0x1105, - CANONICALIZE_PATHS = 0x1106, - SET_DIR_TIMESTAMP = 0x1107, + SCAN_DIRS_ASYNC = 0x1105, + SCAN_FILES_ASYNC = 0x1106, CANCEL_OPERATION = 0x1108, // handle result JUDGE_STATUS = 0x1201, diff --git a/src/framework/common/cs-context.cpp b/src/framework/common/cs-context.cpp index d8ebedd..e744da2 100644 --- a/src/framework/common/cs-context.cpp +++ b/src/framework/common/cs-context.cpp @@ -29,7 +29,8 @@ namespace Csr { CsContext::CsContext() noexcept : askUser(CSR_CS_ASK_USER_NO), coreUsage(CSR_CS_CORE_USAGE_DEFAULT), - isScanOnCloud(false) + isScanOnCloud(false), + isScannedCbRegistered(false) { } @@ -37,8 +38,9 @@ CsContext::CsContext(IStream &stream) { int intAskUser; int intCoreUsage; - Deserializer<std::string, int, int, bool>::Deserialize(stream, - this->popupMessage, intAskUser, intCoreUsage, this->isScanOnCloud); + Deserializer<std::string, int, int, bool, bool>::Deserialize(stream, + this->popupMessage, intAskUser, intCoreUsage, this->isScanOnCloud, + this->isScannedCbRegistered); this->askUser = static_cast<csr_cs_ask_user_e>(intAskUser); this->coreUsage = static_cast<csr_cs_core_usage_e>(intCoreUsage); @@ -46,9 +48,10 @@ CsContext::CsContext(IStream &stream) void CsContext::Serialize(IStream &stream) const { - Serializer<std::string, int, int, bool>::Serialize(stream, + Serializer<std::string, int, int, bool, bool>::Serialize(stream, this->popupMessage, static_cast<int>(this->askUser), - static_cast<int>(this->coreUsage), this->isScanOnCloud); + static_cast<int>(this->coreUsage), this->isScanOnCloud, + this->isScannedCbRegistered); } void CsContext::set(int key, int value) @@ -98,6 +101,10 @@ void CsContext::set(int key, bool value) this->isScanOnCloud = value; break; + case Key::ScannedCbRegistered: + this->isScannedCbRegistered = value; + break; + default: ThrowExc(CSR_ERROR_SERVER, "Invalid key[" << key << "] comes in to set as bool."); } @@ -153,6 +160,10 @@ void CsContext::get(int key, bool &value) const value = this->isScanOnCloud; break; + case Key::ScannedCbRegistered: + value = this->isScannedCbRegistered; + break; + default: ThrowExc(CSR_ERROR_SERVER, "Invalid key[" << key << "] comes in to get as bool."); } diff --git a/src/framework/common/cs-context.h b/src/framework/common/cs-context.h index 0161c26..3ef7647 100644 --- a/src/framework/common/cs-context.h +++ b/src/framework/common/cs-context.h @@ -43,6 +43,7 @@ struct API CsContext : public IContext { CoreUsage = 0x11, // int ScanOnCloud = 0x20, // bool + ScannedCbRegistered = 0x21, }; CsContext() noexcept; @@ -70,6 +71,7 @@ struct API CsContext : public IContext { csr_cs_ask_user_e askUser; csr_cs_core_usage_e coreUsage; bool isScanOnCloud; + bool isScannedCbRegistered; }; } diff --git a/src/framework/common/cs-detected.cpp b/src/framework/common/cs-detected.cpp index 2871f2f..21d7397 100644 --- a/src/framework/common/cs-detected.cpp +++ b/src/framework/common/cs-detected.cpp @@ -63,6 +63,35 @@ void CsDetected::Serialize(IStream &stream) const this->isApp, ts64); } +CsDetected::CsDetected(const CsDetected &other) noexcept : + targetName(other.targetName), + malwareName(other.malwareName), + detailedUrl(other.detailedUrl), + pkgId(other.pkgId), + severity(other.severity), + response(other.response), + isApp(other.isApp), + ts(other.ts) +{ +} + +CsDetected &CsDetected::operator=(const CsDetected &other) noexcept +{ + if (this == &other) + return *this; + + this->targetName = other.targetName; + this->malwareName = other.malwareName; + this->detailedUrl = other.detailedUrl; + this->pkgId = other.pkgId; + this->severity = other.severity; + this->response = other.response; + this->isApp = other.isApp; + this->ts = other.ts; + + return *this; +} + CsDetected::CsDetected(CsDetected &&other) noexcept : targetName(std::move(other.targetName)), malwareName(std::move(other.malwareName)), diff --git a/src/framework/common/cs-detected.h b/src/framework/common/cs-detected.h index 587a3b9..7826dcf 100644 --- a/src/framework/common/cs-detected.h +++ b/src/framework/common/cs-detected.h @@ -41,6 +41,8 @@ struct API CsDetected : public IResult { CsDetected(IStream &); virtual void Serialize(IStream &) const override; + CsDetected(const CsDetected &) noexcept; + CsDetected &operator=(const CsDetected &) noexcept; CsDetected(CsDetected &&) noexcept; CsDetected &operator=(CsDetected &&) noexcept; diff --git a/src/framework/common/dispatcher.cpp b/src/framework/common/dispatcher.cpp index 9f8e023..2982a23 100644 --- a/src/framework/common/dispatcher.cpp +++ b/src/framework/common/dispatcher.cpp @@ -39,4 +39,12 @@ void Dispatcher::connect() Socket::create(this->m_sockId, Socket::Type::CLIENT)); } +int Dispatcher::getFd() const noexcept +{ + if (this->m_connection == nullptr) + return -1; + else + return this->m_connection->getFd(); +} + } // namespace Csr diff --git a/src/framework/common/dispatcher.h b/src/framework/common/dispatcher.h index 9b4f0fb..08a91ee 100644 --- a/src/framework/common/dispatcher.h +++ b/src/framework/common/dispatcher.h @@ -40,12 +40,17 @@ public: Dispatcher(Dispatcher &&) = delete; Dispatcher &operator=(Dispatcher &&) = delete; + int getFd(void) const noexcept; + template<typename Type, typename ...Args> Type methodCall(Args &&...args); template<typename ...Args> void methodPing(Args &&...args); + template<typename Type> + Type receiveEvent(void); + private: void connect(void); @@ -78,4 +83,18 @@ void Dispatcher::methodPing(Args &&...args) this->m_connection->send(BinaryQueue::Serialize(std::forward<Args>(args)...).pop()); } +template<typename Type> +Type Dispatcher::receiveEvent() +{ + this->connect(); + + BinaryQueue q; + q.push(this->m_connection->receive()); + + Type response; + q.Deserialize(response); + + return response; +} + } diff --git a/src/framework/common/mainloop.h b/src/framework/common/mainloop.h index 7a16bab..18b04d2 100644 --- a/src/framework/common/mainloop.h +++ b/src/framework/common/mainloop.h @@ -45,6 +45,9 @@ public: // if timeout is negative value, no timeout on idle. void run(int timeout); + // Moved to public to customize stop condition + void dispatch(int timeout); + void addEventSource(int fd, uint32_t event, Callback &&callback); void removeEventSource(int fd); size_t countEventSource(void) const; @@ -52,7 +55,6 @@ public: void setIdleChecker(std::function<bool()> &&idleChecker); private: - void dispatch(int timeout); bool m_isTimedOut; int m_pollfd; diff --git a/src/framework/common/serialization.h b/src/framework/common/serialization.h index b11baea..6b31678 100644 --- a/src/framework/common/serialization.h +++ b/src/framework/common/serialization.h @@ -325,6 +325,15 @@ struct Deserialization { object.reset(new T(stream)); } + template <typename T> + static void Deserialize(IStream &stream, std::unique_ptr<T> &object) + { + if (stream.empty()) + object.reset(); + else + object.reset(new T(stream)); + } + // char static void Deserialize(IStream &stream, char &value) { @@ -519,6 +528,20 @@ struct Deserialization { } } + template <typename T, typename R, typename A> + static void Deserialize(IStream &stream, std::unique_ptr<std::basic_string<T, R, A>> &str) + { + if (stream.empty()) { + str.reset(); + } else { + int length; + stream.read(sizeof(length), &length); + std::vector<T> buf(length); + stream.read(length * sizeof(T), buf.data()); + str.reset(new std::basic_string<T, R, A>(buf.data(), buf.data() + length)); + } + } + // STL templates // std::list @@ -554,6 +577,16 @@ struct Deserialization { Deserialize(stream, *list); } } + template <typename T> + static void Deserialize(IStream &stream, std::unique_ptr<std::list<T>> &list) + { + if (stream.empty()) { + list.reset(); + } else { + list.reset(new std::list<T>); + Deserialize(stream, *list); + } + } template <typename T> static void Deserialize(IStream &stream, std::set<T> &set) @@ -587,6 +620,16 @@ struct Deserialization { Deserialize(stream, *set); } } + template <typename T> + static void Deserialize(IStream &stream, std::unique_ptr<std::set<T>> &set) + { + if (stream.empty()) { + set.reset(); + } else { + set.reset(new std::set<T>); + Deserialize(stream, *set); + } + } // RawBuffer template <typename A> @@ -617,6 +660,16 @@ struct Deserialization { Deserialize(stream, *vec); } } + template <typename A> + static void Deserialize(IStream &stream, std::unique_ptr<std::vector<unsigned char, A>> &vec) + { + if (stream.empty()) { + vec.reset(); + } else { + vec.reset(new std::vector<unsigned char, A>); + Deserialize(stream, *vec); + } + } // std::vector template <typename T, typename A> diff --git a/src/framework/common/socket.cpp b/src/framework/common/socket.cpp index ceacba8..f698772 100644 --- a/src/framework/common/socket.cpp +++ b/src/framework/common/socket.cpp @@ -161,11 +161,10 @@ RawBuffer Socket::read(void) const size_t total = 0; size_t size = 0; - DEBUG("Read data from stream on socket fd[" << this->m_fd << "]"); - auto bytes = ::read(this->m_fd, &size, sizeof(size)); if (bytes < 0) - ThrowExc(CSR_ERROR_SOCKET, "Socket data size read failed with errno: " << errno); + ThrowExc(CSR_ERROR_SOCKET, "Socket data size read failed on fd[" << this->m_fd << + "] with errno: " << errno); RawBuffer data(size, 0); auto buf = reinterpret_cast<char *>(data.data()); @@ -199,11 +198,10 @@ void Socket::write(const RawBuffer &data) const auto buf = reinterpret_cast<const char *>(data.data()); auto size = data.size(); - DEBUG("Write data to stream on socket fd[" << this->m_fd << "]"); - auto bytes = ::write(this->m_fd, &size, sizeof(size)); if (bytes < 0) - ThrowExc(CSR_ERROR_SOCKET, "Socket data size write failed with errno: " << errno); + ThrowExcWarn(CSR_ERROR_SOCKET, "Socket data size write failed on fd[" << this->m_fd << + "] with errno: " << errno); while (total < size) { bytes = ::write(this->m_fd, buf + total, size - total); @@ -212,7 +210,7 @@ void Socket::write(const RawBuffer &data) const if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) continue; else - ThrowExc(CSR_ERROR_SOCKET, "Socket write failed on fd[" << this->m_fd << + ThrowExcWarn(CSR_ERROR_SOCKET, "Socket write failed on fd[" << this->m_fd << "] with errno: " << errno); } diff --git a/src/framework/db/cache.cpp b/src/framework/db/cache.cpp new file mode 100644 index 0000000..9374bf2 --- /dev/null +++ b/src/framework/db/cache.cpp @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2016 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License + */ +/* + * @file cache.cpp + * @author Sangwan Kwon (sangwan.kwon@samsung.com) + * @version 1.0 + * @brief Cache for hold the data temporally before insert to DB + */ +#include "db/cache.h" +#include "common/audit/logger.h" + +namespace Csr { +namespace Db { + +Cache::Cache(Cache &&rhs) noexcept : + pkgPath(std::move(rhs.pkgPath)), + pkgId(std::move(rhs.pkgId)), + dataVersion(std::move(rhs.dataVersion)), + riskiestPath(std::move(rhs.riskiestPath)), + filePaths(std::move(rhs.filePaths)), + detecteds(std::move(rhs.detecteds)), + riskiest(std::move(rhs.riskiest)), + scanTime(rhs.scanTime) +{ +} + +Cache &Cache::operator=(Cache &&rhs) noexcept +{ + if (this == &rhs) + return *this; + + this->pkgPath = std::move(rhs.pkgPath); + this->pkgId = std::move(rhs.pkgId); + this->dataVersion = std::move(rhs.dataVersion); + this->riskiestPath = std::move(rhs.riskiestPath); + this->filePaths = std::move(rhs.filePaths); + this->detecteds = std::move(rhs.detecteds); + this->riskiest = std::move(rhs.riskiest); + this->scanTime = rhs.scanTime; + + return *this; +} + +} // namespace Db +} // namespace Csr diff --git a/src/framework/db/cache.h b/src/framework/db/cache.h new file mode 100644 index 0000000..6bf2ec4 --- /dev/null +++ b/src/framework/db/cache.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2016 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License + */ +/* + * @file cache.h + * @author Sangwan Kwon (sangwan.kwon@samsung.com) + * @version 1.0 + * @brief Cache for hold the data temporally before insert to DB + */ +#pragma once + +#include <string> +#include <vector> +#include <ctime> + +#include "common/cs-detected.h" + +namespace Csr { +namespace Db { + +struct Cache { +public: + Cache() = default; + virtual ~Cache() = default; + + Cache(const Cache &) = delete; + Cache &operator=(const Cache &) = delete; + Cache(Cache &&) noexcept; + Cache &operator=(Cache &&) noexcept; + + std::string pkgPath; + std::string pkgId; + std::string dataVersion; + std::string riskiestPath; + std::vector<std::string> filePaths; + std::vector<CsDetectedPtr> detecteds; + CsDetectedPtr riskiest; + time_t scanTime; +}; + +} // namespace Db +} // namespace Csr diff --git a/src/framework/db/connection.cpp b/src/framework/db/connection.cpp index cf0a2ce..ab110e5 100644 --- a/src/framework/db/connection.cpp +++ b/src/framework/db/connection.cpp @@ -43,5 +43,14 @@ int Connection::exec(const std::string &query) return ::sqlite3_changes(m_handle); } +void Connection::transactionBegin() +{ + ::sqlite3_exec(m_handle, "BEGIN TRANSACTION;", 0, 0, 0); +} + +void Connection::transactionEnd() +{ + ::sqlite3_exec(m_handle, "END TRANSACTION;", 0, 0, 0); +} } // namespace Db } // namespace Csr diff --git a/src/framework/db/connection.h b/src/framework/db/connection.h index 795a08f..e9e4391 100644 --- a/src/framework/db/connection.h +++ b/src/framework/db/connection.h @@ -36,6 +36,9 @@ public: int exec(const std::string &query); + void transactionBegin(); + void transactionEnd(); + inline long long getLastInsertRowId() const noexcept { return sqlite3_last_insert_rowid(m_handle); diff --git a/src/framework/db/manager.cpp b/src/framework/db/manager.cpp index f2b81ba..79d7454 100644 --- a/src/framework/db/manager.cpp +++ b/src/framework/db/manager.cpp @@ -37,7 +37,8 @@ namespace { enum SchemaVersion : int { NOT_EXIST = 0, _1 = 1, - LATEST = _1 + _2 = 2, + LATEST = _2 }; const std::string SCRIPT_CREATE_SCHEMA = "create_schema"; @@ -113,9 +114,13 @@ Manager::Manager(const std::string &dbfile, const std::string &scriptsDir) : INFO("Database migration! from[" << sv << "] to[" << SchemaVersion::LATEST << "]"); + this->m_conn.exec("PRAGMA foreign_keys = OFF;"); + for (int vi = sv; vi < SchemaVersion::LATEST; ++vi) this->m_conn.exec(this->getMigrationScript(vi).c_str()); + this->m_conn.exec("PRAGMA foreign_keys = ON;"); + this->setSchemaVersion(SchemaVersion::LATEST); } break; @@ -457,31 +462,30 @@ RowShPtr Manager::getWorstByPkgPath(const std::string &pkgPath, time_t since) return row; } -void Manager::insertDetectedFile(const std::string &filepath, const CsDetected &d, - const std::string &dataVersion) +void Manager::insertDetectedFile(const CsDetected &d, const std::string &dataVersion) { std::lock_guard<std::mutex> l(this->m_mutex); - this->insertName(filepath); - this->insertDetected(d, filepath, dataVersion); + this->insertName(d.targetName); + this->insertDetected(d, d.targetName, dataVersion); } -void Manager::insertDetectedFileInApp(const std::string &pkgpath, const std::string &filepath, - const CsDetected &d, const std::string &dataVersion) +void Manager::insertDetectedAppByCloud(const std::string &name, const std::string &pkgId, + const CsDetected &d, const std::string &dataVersion) { std::lock_guard<std::mutex> l(this->m_mutex); - this->insertName(pkgpath); - this->insertDetected(d, filepath, dataVersion); + this->insertName(name); + this->insertDetectedCloud(d, pkgId, name, dataVersion); } -void Manager::insertDetectedAppByCloud(const std::string &name, const std::string &pkgId, - const CsDetected &d, const std::string &dataVersion) +void Manager::insertCache(const Cache &c) { std::lock_guard<std::mutex> l(this->m_mutex); - this->insertName(name); - this->insertDetectedCloud(d, pkgId, name, dataVersion); + this->insertName(c.pkgPath); + for (std::vector<int>::size_type i = 0; i < c.detecteds.size(); ++i) + this->insertDetected(*c.detecteds[i], c.filePaths[i], c.dataVersion); } void Manager::insertName(const std::string &name) @@ -489,6 +493,7 @@ void Manager::insertName(const std::string &name) Statement stmt(this->m_conn, Query::INS_NAME); stmt.bind(name); + stmt.bind(name); stmt.exec(); } @@ -581,5 +586,15 @@ void Manager::deleteDetectedDeprecated(time_t since) stmt2.exec(); } +void Manager::transactionBegin() +{ + this->m_conn.transactionBegin(); +} + +void Manager::transactionEnd() +{ + this->m_conn.transactionEnd(); +} + } // namespace Db } // namespace Csr diff --git a/src/framework/db/manager.h b/src/framework/db/manager.h index d63e05b..325f305 100644 --- a/src/framework/db/manager.h +++ b/src/framework/db/manager.h @@ -29,6 +29,7 @@ #include "db/connection.h" #include "db/row.h" +#include "db/cache.h" #include "common/cs-detected.h" #include <csr-engine-manager.h> @@ -66,10 +67,8 @@ public: RowShPtrs getDetectedByFilepathOnDir(const std::string &dir, time_t since); RowShPtr getWorstByPkgPath(const std::string &pkgPath, time_t since); - void insertDetectedFile(const std::string &filepath, const CsDetected &d, - const std::string &dataVersion); - void insertDetectedFileInApp(const std::string &pkgpath, const std::string &filepath, - const CsDetected &d, const std::string &dataVersion); + void insertCache(const Cache &c); + void insertDetectedFile(const CsDetected &d, const std::string &dataVersion); void insertDetectedAppByCloud(const std::string &name, const std::string &pkgId, const CsDetected &d, const std::string &dataVersion); void insertWorst(const std::string &pkgId, const std::string &name, @@ -80,6 +79,9 @@ public: void deleteDetectedByFilepathOnPath(const std::string &path); void deleteDetectedDeprecated(time_t since); + void transactionBegin(); + void transactionEnd(); + private: RowShPtr getDetectedByNameOnPath(const std::string &path, time_t since); RowShPtr getDetectedCloudByNameOnPath(const std::string &path, time_t since); diff --git a/src/framework/db/query.h b/src/framework/db/query.h index 049434d..38bb4a9 100644 --- a/src/framework/db/query.h +++ b/src/framework/db/query.h @@ -87,7 +87,8 @@ const std::string SEL_WORST_BY_PKGPATH = " where name = ? and detected_time >= ?"; const std::string INS_NAME = - "insert or replace into NAMES(name) values(?)"; + "insert into NAMES(name) select ? where not exists" + " (select 1 from NAMES where name = ?)"; const std::string INS_DETECTED_CLOUD = "insert or replace into DETECTED_MALWARE_CLOUD(idx, pkg_id, data_version," diff --git a/src/framework/service/cs-logic.cpp b/src/framework/service/cs-logic.cpp index b02f263..ed0542e 100644 --- a/src/framework/service/cs-logic.cpp +++ b/src/framework/service/cs-logic.cpp @@ -16,12 +16,14 @@ /* * @file cs-logic.cpp * @author Kyungwook Tak (k.tak@samsung.com) + * @author Sangwan Kwon (sangwan.kwon@samsung.com) * @version 1.0 * @brief */ #include "service/cs-logic.h" #include <utility> +#include <cstdlib> #include <climits> #include <cerrno> #include <unistd.h> @@ -30,6 +32,7 @@ #include "common/audit/logger.h" #include "common/exception.h" +#include "common/async-protocol.h" #include "service/type-converter.h" #include "service/core-usage.h" #include "service/dir-blacklist.h" @@ -132,7 +135,7 @@ FilePtr canonicalizePathWithFile(const std::string &path) { auto resolved = resolvePath(path); - auto fileptr = File::create(path, nullptr); + auto fileptr = File::create(resolved, nullptr); if (!isReadable(fileptr->getName())) ThrowExcWarn(CSR_ERROR_FILE_DO_NOT_EXIST, "File is not readable: " << fileptr->getName()); @@ -140,6 +143,33 @@ FilePtr canonicalizePathWithFile(const std::string &path) return fileptr; } +void eraseSubdirectories(StrSet &dirset) +{ + if (dirset.size() < 2) + return; + + for (auto it = dirset.begin(); it != dirset.end(); ++it) { + auto itsub = it; + ++itsub; + while (true) { + if (itsub == dirset.end()) + break; + + auto itl = it->length(); + auto itsubl = itsub->length(); + + if (itl + 1 >= itsubl || // to short to be sub-directory + itsub->compare(0, itl, *it) != 0 || // prefix isn't matched + (*it != "/" && itsub->at(itl) != '/')) { // has '/' at the end of prefix + ++itsub; + continue; + } + + itsub = dirset.erase(itsub); + } + } +} + } // namespace anonymous CsLogic::CsLogic(const std::shared_ptr<CsLoader> &loader, @@ -176,15 +206,17 @@ RawBuffer CsLogic::scanData(const CsContext &context, const RawBuffer &data) if (result == nullptr) return BinaryQueue::Serialize(CSR_ERROR_NONE).pop(); - auto d = this->convert(result, std::string(), timestamp); + auto malware = this->convert(result, std::string(), timestamp); - return this->handleAskUser(context, d); + return BinaryQueue::Serialize(this->handleAskUser(context, *malware), malware).pop(); } -RawBuffer CsLogic::scanAppOnCloud(const CsContext &context, - const std::string &pkgPath, - const std::string &pkgId) +int CsLogic::scanAppOnCloud(const CsContext &context, const FilePtr &pkgPtr, + CsDetectedPtr &malware) { + const auto &pkgPath = pkgPtr->getName(); + const auto &pkgId = pkgPtr->getAppPkgId(); + CsEngineContext engineContext(this->m_loader); auto &c = engineContext.get(); @@ -194,198 +226,257 @@ RawBuffer CsLogic::scanAppOnCloud(const CsContext &context, this->m_loader->scanAppOnCloud(c, pkgPath, &result); if (!result) - return BinaryQueue::Serialize(CSR_ERROR_NONE).pop(); + return CSR_ERROR_NONE; - auto detected = this->convert(result, pkgPath, timestamp); - detected.isApp = true; - detected.pkgId = pkgId; + malware = this->convert(result, pkgPath, timestamp); + malware->isApp = true; + malware->pkgId = pkgId; - this->m_db->insertDetectedAppByCloud(pkgPath, pkgId, detected, this->m_dataVersion); + this->m_db->insertDetectedAppByCloud(pkgPath, pkgId, *malware, this->m_dataVersion); - return this->handleAskUser(context, detected); + return this->handleAskUser(context, *malware, pkgPtr); } -CsDetectedPtr CsLogic::scanAppDelta(const std::string &pkgPath, const std::string &pkgId, - std::string &riskiestPath) +Db::Cache CsLogic::scanAppDelta(const FilePtr &pkgPtr, const CancelChecker &isCancelled) { - auto starttime = ::time(nullptr); + const auto &pkgPath = pkgPtr->getName(); + const auto &pkgId = pkgPtr->getAppPkgId(); CsEngineContext engineContext(this->m_loader); auto &c = engineContext.get(); auto t = this->m_loader->getEngineLatestUpdateTime(c); auto lastScanTime = this->m_db->getLastScanTime(pkgPath, t); + auto rows = this->m_db->getDetectedByFilepathOnDir(pkgPath, t); + + Db::Cache cache; + cache.pkgPath = pkgPath; + cache.pkgId = pkgId; + cache.scanTime = ::time(nullptr); + cache.dataVersion = this->m_dataVersion; - CsDetectedPtr riskiest; // traverse files in app and take which is more danger than riskiest auto visitor = FsVisitor::create([&](const FilePtr &file) { - DEBUG("Scan file by engine: " << file->getPath()); + if (isCancelled != nullptr) + isCancelled(); + + const auto &path = file->getPath(); + DEBUG("Scan file by engine: " << path); + + // erase newly scanned file from db history if exists + for (auto it = rows.begin(); it != rows.end(); ++it) { + if ((*it)->fileInAppPath == path) { + INFO("Detected malware file is changed, it'll be newly scanned"); + this->m_db->deleteDetectedByFilepathOnPath(path); + rows.erase(it); + break; + } + } auto timestamp = ::time(nullptr); csre_cs_detected_h result = nullptr; - this->m_loader->scanFile(c, file->getPath(), &result); - - if (!result) { - if (lastScanTime != -1) - this->m_db->deleteDetectedByFilepathOnPath(file->getPath()); + this->m_loader->scanFile(c, path, &result); + if (!result) return; - } - INFO("New malware detected on file: " << file->getPath()); + INFO("New malware detected on file: " << path); auto candidate = this->convert(result, pkgPath, timestamp); - candidate.isApp = true; - candidate.pkgId = pkgId; - - this->m_db->insertDetectedFileInApp(pkgPath, file->getPath(), candidate, - this->m_dataVersion); - - if (!riskiest) { - riskiest.reset(new CsDetected(std::move(candidate))); - riskiestPath = file->getPath(); - } else if (*riskiest < candidate) { - *riskiest = std::move(candidate); - riskiestPath = file->getPath(); + candidate->isApp = true; + candidate->pkgId = pkgId; + + if (!cache.riskiest) { + cache.riskiest.reset(new CsDetected(*candidate)); + cache.riskiestPath = path; + } else if (*(cache.riskiest) < *candidate) { + *(cache.riskiest) = *candidate; + cache.riskiestPath = path; } + + cache.filePaths.push_back(path); + cache.detecteds.emplace_back(std::move(candidate)); }, pkgPath, false, lastScanTime); visitor->run(); - this->m_db->insertLastScanTime(pkgPath, this->m_dataVersion, starttime); - - return riskiest; + return cache; } -RawBuffer CsLogic::scanApp(const CsContext &context, const FilePtr &pkgPtr) +int CsLogic::scanApp(const CsContext &context, const FilePtr &pkgPtr, + CsDetectedPtr &malware, const CancelChecker &isCancelled) { const auto &pkgPath = pkgPtr->getName(); const auto &pkgId = pkgPtr->getAppPkgId(); if (context.isScanOnCloud && this->m_loader->scanAppOnCloudSupported()) - return this->scanAppOnCloud(context, pkgPath, pkgId); + return this->scanAppOnCloud(context, pkgPtr, malware); CsEngineContext engineContext(this->m_loader); auto since = this->m_loader->getEngineLatestUpdateTime(engineContext.get()); - // old history + // snapshot of history before start delta scan auto history = this->m_db->getWorstByPkgPath(pkgPath, since); - // riskiest detected among newly scanned files - std::string riskiestPath; - auto riskiest = this->scanAppDelta(pkgPath, pkgId, riskiestPath); - // history after delta scan. if worst file is changed, it's rescanned in scanAppDelta - // and deleted from db if it's cured. if history != nullptr && after == nullptr, - // it means worst detected item is cured anyway. - auto after = this->m_db->getWorstByPkgPath(pkgPath, since); - if (history && after && riskiest) { - if (*history < *riskiest) { - INFO("worst case is remained but the more worst newly detected. on pkg[" << - pkgPath << "]"); - if (history->isIgnored) - this->m_db->updateIgnoreFlag(pkgPath, false); - - this->m_db->insertWorst(pkgId, pkgPath, riskiestPath); - return this->handleAskUser(context, *riskiest); - } else { - INFO("worst case is remained and can be re-used on pkg[" << pkgPath << "]"); - if (history->isIgnored) - return BinaryQueue::Serialize(CSR_ERROR_NONE).pop(); - - return this->handleAskUser(context, *history); - } - } else if (history && after && !riskiest) { - INFO("worst case is remained and NO new detected. history can be re-used. " - "on pkg[" << pkgPath << "]"); + // riskiest detected among newly scanned files + auto cache = this->scanAppDelta(pkgPtr, isCancelled); + + // history after delta scan. + // if worst file is changed, it's rescanned in scanAppDelta and row deleted in db + auto isHistoryDeleted = !this->m_db->getWorstByPkgPath(pkgPath, since); + + Db::RowShPtr jWorse; + auto &riskiest = cache.riskiest; + INFO("start to judge scan stage on pkg[" << pkgPath << "]"); + switch (this->judgeScanStage(riskiest, history, isHistoryDeleted, since, jWorse)) { + case CsLogic::ScanStage::NEW_RISKIEST : + this->m_db->transactionBegin(); + this->m_db->insertCache(cache); + this->m_db->insertWorst(pkgId, pkgPath, cache.riskiestPath); + this->m_db->updateIgnoreFlag(pkgPath, false); + this->m_db->insertLastScanTime(pkgPath, cache.dataVersion, cache.scanTime); + this->m_db->transactionEnd(); + + malware = std::move(riskiest); + return this->handleAskUser(context, *malware, pkgPtr); + + case CsLogic::ScanStage::NEW_RISKIEST_KEEP_FLAG : + this->m_db->transactionBegin(); + this->m_db->insertCache(cache); + this->m_db->insertWorst(pkgId, pkgPath, cache.riskiestPath); + this->m_db->insertLastScanTime(pkgPath, cache.dataVersion, cache.scanTime); + this->m_db->transactionEnd(); if (history->isIgnored) - return BinaryQueue::Serialize(CSR_ERROR_NONE).pop(); - - return this->handleAskUser(context, *history); - } else if (history && !after && riskiest) { - INFO("worst case is deleted but new detected. we have to find out " - "worse case in db and compare it with riskiest first. on pkg[" << pkgPath << - "]"); - Db::RowShPtr worse; - since = this->m_loader->getEngineLatestUpdateTime(engineContext.get()); - for (auto &row : this->m_db->getDetectedByFilepathOnDir(pkgPath, since)) - if (!worse || *worse < *row) - worse = std::move(row); - - if (!worse) { - INFO("No detected malware found in db.... Newly detected malware is removed by " - "other client. Handle it as fully clean case."); - return BinaryQueue::Serialize(CSR_ERROR_NONE).pop(); - } + return CSR_ERROR_NONE; - if (*riskiest < *worse) { - INFO("worse case in db is worse than riskiest. on pkg[" << pkgPath << "]"); - riskiestPath = worse->fileInAppPath; - *riskiest = std::move(*worse); - } + malware = std::move(riskiest); + return this->handleAskUser(context, *malware, pkgPtr); - if (*history < *riskiest) { - INFO("worst case is deleted but the more worst newly detected. on pkg[" << - pkgPath << "]"); - if (history->isIgnored) - this->m_db->updateIgnoreFlag(pkgPath, false); + case CsLogic::ScanStage::HISTORY_RISKIEST : + this->m_db->transactionBegin(); + this->m_db->insertCache(cache); + this->m_db->insertWorst(pkgId, pkgPath, history->fileInAppPath); + this->m_db->insertLastScanTime(pkgPath, cache.dataVersion, cache.scanTime); + this->m_db->transactionEnd(); + if (history->isIgnored) + return CSR_ERROR_NONE; - this->m_db->insertWorst(pkgId, pkgPath, riskiestPath); + malware.reset(new CsDetected()); + *malware = *history; + return this->handleAskUser(context, *malware, pkgPtr); - return this->handleAskUser(context, *riskiest); - } else { - INFO("worst case is deleted but same or less level newly detected. on pkg[" << - pkgPath << "]"); - this->m_db->insertWorst(pkgId, pkgPath, riskiestPath); + case CsLogic::ScanStage::WORSE_RISKIEST : + this->m_db->transactionBegin(); + this->m_db->insertCache(cache); + this->m_db->insertWorst(pkgId, pkgPath, jWorse->fileInAppPath); + this->m_db->insertLastScanTime(pkgPath, cache.dataVersion, cache.scanTime); + this->m_db->transactionEnd(); + if (jWorse->isIgnored) + return CSR_ERROR_NONE; - if (history->isIgnored) - return BinaryQueue::Serialize(CSR_ERROR_NONE).pop(); + malware.reset(new CsDetected()); + *malware = *jWorse; + return this->handleAskUser(context, *malware, pkgPtr); - return this->handleAskUser(context, *riskiest); - } - } else if (history && !after && !riskiest) { - since = this->m_loader->getEngineLatestUpdateTime(engineContext.get()); - auto rows = this->m_db->getDetectedByFilepathOnDir(pkgPath, since); + case CsLogic::ScanStage::NO_DETECTED : + this->m_db->insertLastScanTime(pkgPath, cache.dataVersion, cache.scanTime); + return CSR_ERROR_NONE; - if (!rows.empty()) { - INFO("worst case is deleted cascadingly and NO new detected and " - "worse case exist on pkg[" << pkgPath << "]. insert it to worst."); - Db::RowShPtr worse; - for (auto &row : rows) - if (!worse || *worse < *row) - worse = std::move(row); + default: + ThrowExc(CSR_ERROR_SERVER, "Invalid scan app status."); + } +} - if (worse) { - this->m_db->insertWorst(pkgId, pkgPath, worse->fileInAppPath); +Db::RowShPtr CsLogic::getWorseByPkgPath(const std::string &pkgPath, time_t since) +{ + Db::RowShPtr worse; + for (auto &row : this->m_db->getDetectedByFilepathOnDir(pkgPath, since)) + if (!worse || *worse < *row) + worse = std::move(row); - if (worse->isIgnored) - return BinaryQueue::Serialize(CSR_ERROR_NONE).pop(); + return worse; +} - return this->handleAskUser(context, *worse); +CsLogic::ScanStage CsLogic::judgeScanStage(const CsDetectedPtr &riskiest, + const Db::RowShPtr &history, bool isHistoryDeleted, + time_t since, Db::RowShPtr &jWorse) +{ + if (riskiest == nullptr) { + if (history == nullptr) { + INFO("no new detected and no history."); + return ScanStage::NO_DETECTED; + } else if (isHistoryDeleted) { + jWorse = this->getWorseByPkgPath(history->targetName, since); + if (jWorse == nullptr) { + INFO("Worst case(history) deleted and no worse and no new malware."); + this->m_db->deleteDetectedByNameOnPath(history->targetName); + return ScanStage::NO_DETECTED; + } else { + INFO("Worst case(history) deleted and worse exist and no new malware."); + return ScanStage::WORSE_RISKIEST; + } + } else { + INFO("Worst case(history) isn't deleted and no new detected. History can be " + "re-used."); + return ScanStage::HISTORY_RISKIEST; + } + } else if (history == nullptr) { + INFO("No history have been existed and new malware detected!"); + return ScanStage::NEW_RISKIEST; + } else if (isHistoryDeleted) { + jWorse = this->getWorseByPkgPath(history->targetName, since); + if (jWorse == nullptr) { + if (*history < *riskiest) { + INFO("Worst case(history) changed/removed and no worse. Newly detected " + "malware is more risky than history!"); + return ScanStage::NEW_RISKIEST; + } else { + INFO("Worst case(history) changed/removed and no worse. Newly detected " + "malware isn't more risky than history. Keep ignored flag."); + return ScanStage::NEW_RISKIEST_KEEP_FLAG; + } + } else { + if (*jWorse < *riskiest) { + if (*history < *riskiest) { + INFO("Worst case(history) deleted and 'worse <= history < riskiest'"); + return ScanStage::NEW_RISKIEST; + } else { + INFO("Worst case(history) deleted and 'worse < riskiest <= history'"); + return ScanStage::NEW_RISKIEST_KEEP_FLAG; + } + } else { + INFO("Worst case(history) deleted and 'riskiest <= worse <= history'"); + return ScanStage::WORSE_RISKIEST; } } - - INFO("worst case is deleted cascadingly and NO new detected and " - "NO worse case. the pkg[" << pkgPath << "] is clean."); - - this->m_db->deleteDetectedByNameOnPath(pkgPath); - return BinaryQueue::Serialize(CSR_ERROR_NONE).pop(); - } else if (!history && riskiest) { - INFO("no history and new detected"); - this->m_db->insertWorst(pkgId, pkgPath, riskiestPath); - - return this->handleAskUser(context, *riskiest); } else { - DEBUG("no history and no new detected"); - return BinaryQueue::Serialize(CSR_ERROR_NONE).pop(); + if (*history < *riskiest) { + INFO("Worst case(history) remained and 'history < riskiest'"); + return ScanStage::NEW_RISKIEST; + } else { + INFO("Worst case(history) remained and 'riskiest <= history'. " + "History can be re-used"); + return ScanStage::HISTORY_RISKIEST; + } } } -RawBuffer CsLogic::scanFileWithoutDelta(const CsContext &context, - const std::string &filepath, FilePtr &&fileptr) +int CsLogic::scanFileInternal(const CsContext &context, const FilePtr &target, + CsDetectedPtr &malware, const CancelChecker &isCancelled) { - if (isInBlackList(filepath)) - return BinaryQueue::Serialize(CSR_ERROR_NONE).pop(); + if (target->isInApp()) + return this->scanApp(context, target, malware, isCancelled); + + const auto &name = target->getName(); + + if (target->isDir()) + ThrowExc(CSR_ERROR_FILE_SYSTEM, "file type shouldn't be directory: " << name); + + DEBUG("Scan request on file: " << name); + + if (isInBlackList(name)) + return CSR_ERROR_NONE; CsEngineContext engineContext(this->m_loader); auto &c = engineContext.get(); @@ -393,19 +484,29 @@ RawBuffer CsLogic::scanFileWithoutDelta(const CsContext &context, auto timestamp = ::time(nullptr); csre_cs_detected_h result = nullptr; - this->m_loader->scanFile(c, filepath, &result); + this->m_loader->scanFile(c, name, &result); // detected handle is null if it's safe if (result == nullptr) - return BinaryQueue::Serialize(CSR_ERROR_NONE).pop(); + return CSR_ERROR_NONE; - INFO("New malware detected on file: " << filepath); + INFO("Malware detected on file: " << name); - auto d = this->convert(result, filepath, timestamp); + malware = this->convert(result, name, timestamp); - this->m_db->insertDetectedFile(d.targetName, d, this->m_dataVersion); + // check malware detected history for inherit ignored flag + auto since = this->m_loader->getEngineLatestUpdateTime(c); + auto history = this->m_db->getDetectedAllByNameOnPath(name, since); - return this->handleAskUser(context, d, std::move(fileptr)); + this->m_db->insertDetectedFile(*malware, this->m_dataVersion); + + if (history != nullptr && history->isIgnored && !(*malware > *history)) { + INFO("Ignore malware on file: " << name); + malware.reset(); + return CSR_ERROR_NONE; + } else { + return this->handleAskUser(context, *malware, target); + } } RawBuffer CsLogic::scanFile(const CsContext &context, const std::string &filepath) @@ -415,100 +516,152 @@ RawBuffer CsLogic::scanFile(const CsContext &context, const std::string &filepat setCoreUsage(context.coreUsage); - auto target = canonicalizePathWithFile(filepath); + CsDetectedPtr malware; + auto ret = this->scanFileInternal(context, canonicalizePathWithFile(filepath), malware); + if (malware != nullptr) + return BinaryQueue::Serialize(ret, malware).pop(); + else + return BinaryQueue::Serialize(ret).pop(); +} - if (target->isInApp()) - return this->scanApp(context, target); +RawBuffer CsLogic::scanFilesAsync(const ConnShPtr &conn, const CsContext &context, + StrSet &paths, const CancelChecker &isCancelled) +{ + if (this->m_db->getEngineState(CSR_ENGINE_CS) != CSR_STATE_ENABLE) + ThrowExc(CSR_ERROR_ENGINE_DISABLED, "engine is disabled"); - const auto &name = target->getName(); + StrSet canonicalized; - if (target->isDir()) - ThrowExc(CSR_ERROR_FILE_SYSTEM, "file type shouldn't be directory: " << name); + for (const auto &path : paths) { + isCancelled(); - DEBUG("Scan request on file: " << name); + FilePtr target; + try { + target = canonicalizePathWithFile(path); - CsEngineContext engineContext(this->m_loader); - auto since = this->m_loader->getEngineLatestUpdateTime(engineContext.get()); - auto history = this->m_db->getDetectedAllByNameOnPath(name, since); + if (canonicalized.find(target->getName()) != canonicalized.end()) + continue; - if (history == nullptr) { - DEBUG("No history exist on target. Newly scan needed: " << name); - return this->scanFileWithoutDelta(context, name, std::move(target)); - } else if (target->isModifiedSince(history->ts)) { - DEBUG("file[" << name << "] is modified since the detected time. " - "let's remove history and re-scan"); - this->m_db->deleteDetectedByNameOnPath(name); - return this->scanFileWithoutDelta(context, name, std::move(target)); - } + INFO("Insert to canonicalized list: " << target->getName()); + canonicalized.insert(target->getName()); + } catch (const Exception &e) { + if (e.error() == CSR_ERROR_FILE_DO_NOT_EXIST || + e.error() == CSR_ERROR_FILE_SYSTEM || + e.error() == CSR_ERROR_PERMISSION_DENIED) { + WARN("File-system & permission related exception occured while getting " + "canonicalize path of path: " << path << " " << e.what() << + ". Ignore this exception."); + continue; + } else { + ERROR("Non-file-system related exception occured while getting " + "canonicalize path of path: " << path << " " << e.what()); + throw; + } + } - DEBUG("Usable scan history exist on file: " << name); + CsDetectedPtr malware; + auto retcode = this->scanFileInternal(context, target, malware, isCancelled); + + switch (retcode) { + case CSR_ERROR_NONE: + case CSR_ERROR_FILE_DO_NOT_EXIST: + case CSR_ERROR_FILE_CHANGED: + case CSR_ERROR_FILE_SYSTEM: + if (malware != nullptr) { + conn->send(BinaryQueue::Serialize(ASYNC_EVENT_MALWARE_DETECTED).pop()); + conn->send(BinaryQueue::Serialize(malware).pop()); + } else if (context.isScannedCbRegistered) { + conn->send(BinaryQueue::Serialize(ASYNC_EVENT_MALWARE_NONE).pop()); + conn->send(BinaryQueue::Serialize(target->getName()).pop()); + } - if (history->isIgnored) - return BinaryQueue::Serialize(CSR_ERROR_NONE).pop(); + break; + + default: + ThrowExc(retcode, "Error on async scanning: " << retcode); + } + } - return this->handleAskUser(context, *history); + return BinaryQueue::Serialize(ASYNC_EVENT_COMPLETE).pop(); } -// Application in input param directory will be treated as one item. -// Application base directory path is inserted to file set. -// e.g., input param dir : "/opt/usr" (applications in "/opt/usr/apps") -// ls /opt/usr/ : -// /opt/usr/file-not-in-app1 -// /opt/usr/file-not-in-app2 -// /opt/usr/apps/org.tizen.tutorial -// /opt/usr/apps/org.tizen.tutorial/file-in-app1 -// /opt/usr/apps/org.tizen.tutorial/file-in-app2 -// /opt/usr/apps/org.tizen.message/file-in-app1 -// /opt/usr/apps/org.tizen.message/file-in-app2 -// /opt/usr/apps/org.tizen.flash/file-in-app1 -// /opt/usr/apps/org.tizen.flash/file-in-app2 -// -// and detected history exist on... -// /opt/usr/apps/org.tizen.message/file-in-app2 -// /opt/usr/apps/org.tizen.flash (If target name is app base directory path, -// it's detected by scan on cloud) -// -// output scannable file set will be: -// 1) /opt/usr/file-not-in-app1 -// 2) /opt/usr/file-not-in-app2 -// 3) /opt/usr/apps/org.tizen.tutorial (app base directory path) -// 4) /opt/usr/apps/org.tizen.message (app base directory path) -// 5) /opt/usr/apps/org.tizen.flash (app base directory path) -// % items which has detected history is included in list as well. -RawBuffer CsLogic::getScannableFiles(const std::string &dir, const std::function<void()> &isCancelled) +RawBuffer CsLogic::scanDirsAsync(const ConnShPtr &conn, const CsContext &context, + StrSet &paths, const CancelChecker &isCancelled) { if (this->m_db->getEngineState(CSR_ENGINE_CS) != CSR_STATE_ENABLE) ThrowExc(CSR_ERROR_ENGINE_DISABLED, "engine is disabled"); - auto targetdir = canonicalizePath(dir, true); + StrSet dirs; - CsEngineContext csEngineContext(this->m_loader); - auto since = this->m_loader->getEngineLatestUpdateTime(csEngineContext.get()); + for (const auto &path : paths) { + isCancelled(); - auto lastScanTime = this->m_db->getLastScanTime(targetdir, since); + try { + auto target = canonicalizePath(path, true); - StrSet fileset; + if (dirs.find(target) == dirs.end()) { + INFO("Insert to canonicalized list: " << target); + dirs.emplace(std::move(target)); + } + } catch (const Exception &e) { + if (e.error() == CSR_ERROR_FILE_DO_NOT_EXIST || + e.error() == CSR_ERROR_FILE_SYSTEM || + e.error() == CSR_ERROR_PERMISSION_DENIED) { + WARN("File-system & permission related exception occured while getting " + "canonicalize path of path: " << path << " " << e.what() << + ". Ignore this exception."); + continue; + } else { + ERROR("Non-file-system related exception occured while getting " + "canonicalize path of path: " << path << " " << e.what()); + throw; + } + } + } - auto visitor = FsVisitor::create([&](const FilePtr &file) { - isCancelled(); + eraseSubdirectories(dirs); - DEBUG("Scannable item: " << file->getName()); - fileset.insert(file->getName()); - }, targetdir, true, lastScanTime); + CsEngineContext engineContext(this->m_loader); + auto t = this->m_loader->getEngineLatestUpdateTime(engineContext.get()); - visitor->run(); + INFO("Start async scanning!!!!!"); - isCancelled(); + StrSet malwareList; + for (const auto &dir : dirs) { + INFO("Start async scanning for dir: " << dir); - if (lastScanTime != -1) { - // for case: scan history exist and not modified. - for (auto &row : this->m_db->getDetectedAllByNameOnDir(targetdir, since)) { + for (auto &row : this->m_db->getDetectedAllByNameOnDir(dir, t)) { isCancelled(); try { auto fileptr = File::create(row->targetName, nullptr); - fileset.insert(fileptr->getName()); + CsDetectedPtr malware; + auto retcode = this->scanFileInternal(context, fileptr, malware, isCancelled); + + switch (retcode) { + case CSR_ERROR_NONE: + case CSR_ERROR_FILE_DO_NOT_EXIST: + case CSR_ERROR_FILE_CHANGED: + case CSR_ERROR_FILE_SYSTEM: + if (malware != nullptr) { + conn->send(BinaryQueue::Serialize(ASYNC_EVENT_MALWARE_DETECTED).pop()); + conn->send(BinaryQueue::Serialize(malware).pop()); + malwareList.insert(row->targetName); + } else if (context.isScannedCbRegistered) { + conn->send(BinaryQueue::Serialize(ASYNC_EVENT_MALWARE_NONE).pop()); + conn->send(BinaryQueue::Serialize(row->targetName).pop()); + this->m_db->deleteDetectedByNameOnPath(row->targetName); + } + + break; + + default: + ERROR("Error on rescanning detected malwares in db: " << retcode << + " file: " << fileptr->getName()); + this->m_db->deleteDetectedByNameOnPath(row->targetName); + return BinaryQueue::Serialize(retcode).pop(); + } } catch (const Exception &e) { if (e.error() == CSR_ERROR_FILE_DO_NOT_EXIST || e.error() == CSR_ERROR_FILE_SYSTEM) @@ -517,35 +670,57 @@ RawBuffer CsLogic::getScannableFiles(const std::string &dir, const std::function throw; } } - } - return BinaryQueue::Serialize(CSR_ERROR_NONE, fileset).pop(); -} + INFO("detected malwares rescanning done from db."); -RawBuffer CsLogic::canonicalizePaths(const StrSet &paths) -{ - if (this->m_db->getEngineState(CSR_ENGINE_CS) != CSR_STATE_ENABLE) - ThrowExc(CSR_ERROR_ENGINE_DISABLED, "engine is disabled"); - - StrSet canonicalized; + auto startTime = ::time(nullptr); + auto lastScanTime = this->m_db->getLastScanTime(dir, t); + auto visitor = FsVisitor::create([&](const FilePtr &file) { + isCancelled(); - for (const auto &path : paths) { - auto target = canonicalizePath(path, true); + CsDetectedPtr malware; + auto retcode = this->scanFileInternal(context, file, malware, isCancelled); + + DEBUG("scanFileInternal done. file: " << file->getName() << + " retcode: " << retcode); + + switch (retcode) { + case CSR_ERROR_NONE: + case CSR_ERROR_FILE_DO_NOT_EXIST: + case CSR_ERROR_FILE_CHANGED: + case CSR_ERROR_FILE_SYSTEM: + if (malware != nullptr) { + auto it = malwareList.find(file->getName()); + if (it != malwareList.end()) { + malwareList.erase(it); + break; + } + + DEBUG("Malware detected!!!"); + conn->send(BinaryQueue::Serialize(ASYNC_EVENT_MALWARE_DETECTED).pop()); + conn->send(BinaryQueue::Serialize(malware).pop()); + } else if (context.isScannedCbRegistered) { + DEBUG("File scanned!!!"); + conn->send(BinaryQueue::Serialize(ASYNC_EVENT_MALWARE_NONE).pop()); + conn->send(BinaryQueue::Serialize(file->getName()).pop()); + } + + break; + + default: + ThrowExc(retcode, "Error on async scanning: " << retcode); + } + }, dir, true, lastScanTime); - if (canonicalized.find(target) == canonicalized.end()) { - INFO("Insert to canonicalized list: " << target); - canonicalized.emplace(std::move(target)); - } - } + visitor->run(); - return BinaryQueue::Serialize(CSR_ERROR_NONE, canonicalized).pop(); -} + this->m_db->insertLastScanTime(dir, this->m_dataVersion, startTime); -RawBuffer CsLogic::setDirTimestamp(const std::string &dir, time_t ts) -{ - this->m_db->insertLastScanTime(dir, this->m_dataVersion, ts); + if (lastScanTime == -1) + continue; + } - return BinaryQueue::Serialize(CSR_ERROR_NONE).pop(); + return BinaryQueue::Serialize(ASYNC_EVENT_COMPLETE).pop(); } RawBuffer CsLogic::judgeStatus(const std::string &filepath, csr_cs_action_e action) @@ -704,11 +879,11 @@ RawBuffer CsLogic::getIgnoredList(const StrSet &_dirSet) return BinaryQueue::Serialize(CSR_ERROR_NONE, rows).pop(); } -RawBuffer CsLogic::handleAskUser(const CsContext &c, CsDetected &d, FilePtr &&fileptr) +int CsLogic::handleAskUser(const CsContext &c, CsDetected &d, const FilePtr &fileptr) { if (c.askUser == CSR_CS_ASK_USER_NO) { d.response = CSR_CS_USER_RESPONSE_USER_NOT_ASKED; - return BinaryQueue::Serialize(CSR_ERROR_NONE, d).pop(); + return CSR_ERROR_NONE; } Ui::CommandId cid; @@ -743,57 +918,53 @@ RawBuffer CsLogic::handleAskUser(const CsContext &c, CsDetected &d, FilePtr &&fi auto r = askUser.cs(cid, c.popupMessage, d); if (r == -1) { ERROR("Failed to get user response by popup service for target: " << d.targetName); - return BinaryQueue::Serialize(CSR_ERROR_USER_RESPONSE_FAILED, d).pop(); + return CSR_ERROR_USER_RESPONSE_FAILED; } d.response = r; - if (d.response == CSR_CS_USER_RESPONSE_REMOVE && !d.targetName.empty()) { - try { - FilePtr _fileptr; - if (fileptr) - _fileptr = std::move(fileptr); - else - _fileptr = File::create(d.targetName, nullptr); + if (d.response != CSR_CS_USER_RESPONSE_REMOVE || d.targetName.empty()) + return CSR_ERROR_NONE; - _fileptr->remove(); - } catch (const Exception &e) { - if (e.error() == CSR_ERROR_FILE_DO_NOT_EXIST) - WARN("File already removed.: " << d.targetName); - else if (e.error() == CSR_ERROR_FILE_SYSTEM) - WARN("File type is changed, considered as different file: " << - d.targetName); - else if (e.error() == CSR_ERROR_REMOVE_FAILED) - return BinaryQueue::Serialize(CSR_ERROR_REMOVE_FAILED, d).pop(); - else - throw; - } - - this->m_db->deleteDetectedByNameOnPath(d.targetName); + try { + if (fileptr != nullptr) + fileptr->remove(); + else + File::create(d.targetName, nullptr)->remove(); + } catch (const Exception &e) { + if (e.error() == CSR_ERROR_FILE_DO_NOT_EXIST) + WARN("File already removed.: " << d.targetName); + else if (e.error() == CSR_ERROR_FILE_SYSTEM) + WARN("File type is changed, considered as different file: " << d.targetName); + else if (e.error() == CSR_ERROR_REMOVE_FAILED) + return CSR_ERROR_REMOVE_FAILED; + else + throw; } - return BinaryQueue::Serialize(CSR_ERROR_NONE, d).pop(); + this->m_db->deleteDetectedByNameOnPath(d.targetName); + return CSR_ERROR_NONE; } -CsDetected CsLogic::convert(csre_cs_detected_h &result, const std::string &targetName, +CsDetectedPtr CsLogic::convert(csre_cs_detected_h &result, const std::string &targetName, time_t timestamp) { DEBUG("convert engine result handle to CsDetected start"); - CsDetected d; + CsDetectedPtr malware(new CsDetected()); - d.targetName = targetName; + malware->targetName = targetName; csre_cs_severity_level_e eseverity = CSRE_CS_SEVERITY_LOW; this->m_loader->getSeverity(result, &eseverity); - this->m_loader->getMalwareName(result, d.malwareName); - this->m_loader->getDetailedUrl(result, d.detailedUrl); + this->m_loader->getMalwareName(result, malware->malwareName); + this->m_loader->getDetailedUrl(result, malware->detailedUrl); - d.ts = timestamp; - d.severity = Csr::convert(eseverity); + malware->ts = timestamp; + malware->severity = Csr::convert(eseverity); - return d; + return malware; } } diff --git a/src/framework/service/cs-logic.h b/src/framework/service/cs-logic.h index 86ce3d3..0cf2f10 100644 --- a/src/framework/service/cs-logic.h +++ b/src/framework/service/cs-logic.h @@ -16,6 +16,7 @@ /* * @file cs-logic.h * @author Kyungwook Tak (k.tak@samsung.com) + * @author Sangwan Kwon (sangwan.kwon@samsung.com) * @version 1.0 * @brief */ @@ -30,6 +31,7 @@ #include "common/cs-context.h" #include "common/cs-detected.h" #include "db/manager.h" +#include "db/cache.h" #include "service/cs-loader.h" #include "service/file-system.h" #include "service/logic.h" @@ -40,15 +42,18 @@ namespace Csr { class CsLogic : public Logic { public: + using CancelChecker = std::function<void()>; + CsLogic(const std::shared_ptr<CsLoader> &loader, const std::shared_ptr<Db::Manager> &db); virtual ~CsLogic() = default; RawBuffer scanData(const CsContext &context, const RawBuffer &data); RawBuffer scanFile(const CsContext &context, const std::string &filepath); - RawBuffer getScannableFiles(const std::string &dir, const std::function<void()> &isCancelled); - RawBuffer canonicalizePaths(const StrSet &paths); - RawBuffer setDirTimestamp(const std::string &dir, time_t ts); + RawBuffer scanFilesAsync(const ConnShPtr &conn, const CsContext &context, + StrSet &paths, const CancelChecker &isCancelled); + RawBuffer scanDirsAsync(const ConnShPtr &conn, const CsContext &context, + StrSet &paths, const CancelChecker &isCancelled); RawBuffer judgeStatus(const std::string &filepath, csr_cs_action_e action); RawBuffer getDetected(const std::string &filepath); RawBuffer getDetectedList(const StrSet &dirSet); @@ -56,24 +61,34 @@ public: RawBuffer getIgnoredList(const StrSet &dirSet); private: - RawBuffer scanApp(const CsContext &context, const FilePtr &pkgPtr); - RawBuffer scanAppOnCloud(const CsContext &context, const std::string &pkgPath, - const std::string &pkgId); - CsDetectedPtr scanAppDelta(const std::string &pkgPath, const std::string &pkgId, - std::string &riskiestPath); + int scanFileInternal(const CsContext &context, const FilePtr &target, CsDetectedPtr &malware, const CancelChecker &isCancelled = nullptr); + int scanApp(const CsContext &context, const FilePtr &pkgPtr, CsDetectedPtr &malware, const CancelChecker &isCancelled = nullptr); + Db::Cache scanAppDelta(const FilePtr &pkgPtr, const CancelChecker &isCancelled = nullptr); + int scanAppOnCloud(const CsContext &context, const FilePtr &pkgPtr, CsDetectedPtr &malware); + + CsDetectedPtr convert(csre_cs_detected_h &result, const std::string &targetName, + time_t timestamp); + int handleAskUser(const CsContext &c, CsDetected &d, const FilePtr &fileptr = nullptr); - RawBuffer scanFileWithoutDelta(const CsContext &context, const std::string &filepath, - FilePtr &&fileptr); + Db::RowShPtr getWorseByPkgPath(const std::string &pkgPath, time_t since); - CsDetected convert(csre_cs_detected_h &result, const std::string &targetName, - time_t timestamp); - RawBuffer handleAskUser(const CsContext &c, CsDetected &d, - FilePtr &&fileptr = nullptr); + enum class ScanStage : int { + NEW_RISKIEST = 0x1001, + NEW_RISKIEST_KEEP_FLAG = 0x1002, + HISTORY_RISKIEST = 0x1003, + WORSE_RISKIEST = 0x1004, + NO_DETECTED = 0x1005 + }; + + CsLogic::ScanStage judgeScanStage(const CsDetectedPtr &riskiest, + const Db::RowShPtr &history, bool isHistoryDeleted, + time_t since, Db::RowShPtr &jWorse); std::shared_ptr<CsLoader> m_loader; std::shared_ptr<Db::Manager> m_db; std::string m_dataVersion; + }; } diff --git a/src/framework/service/exception.cpp b/src/framework/service/exception.cpp index 7e5bf95..5019fde 100644 --- a/src/framework/service/exception.cpp +++ b/src/framework/service/exception.cpp @@ -26,6 +26,8 @@ #include "common/audit/logger.h" #include "common/binary-queue.h" +#include "common/async-protocol.h" + #include <csr-error.h> namespace Csr { @@ -35,7 +37,12 @@ RawBuffer exceptionGuard(const std::function<RawBuffer()> &func) try { return func(); } catch (const Exception &e) { - ERROR("Exception caught. code: " << e.error() << " message: " << e.what()); + if (e.error() == CSR_ERROR_SOCKET) + WARN("Socket error. Client might cancel async scan or crashed: " << e.what()); + else if (e.error() == ASYNC_EVENT_CANCEL) + INFO("Async operation cancel exception: " << e.what()); + else + ERROR("Exception caught. code: " << e.error() << " message: " << e.what()); return BinaryQueue::Serialize(e.error()).pop(); } catch (const std::invalid_argument &e) { ERROR("Invalid argument: " << e.what()); diff --git a/src/framework/service/file-system.cpp b/src/framework/service/file-system.cpp index 934654a..71a1528 100644 --- a/src/framework/service/file-system.cpp +++ b/src/framework/service/file-system.cpp @@ -246,8 +246,8 @@ FsVisitorPtr FsVisitor::create(TargetHandler &&targetHandler, isBasedOnName, modifiedSince)); } -FsVisitor::FsVisitor(TargetHandler &&targetHandler, - const std::string &dirpath, bool isBasedOnName, time_t modifiedSince) : +FsVisitor::FsVisitor(TargetHandler &&targetHandler, const std::string &dirpath, + bool isBasedOnName, time_t modifiedSince) : m_targetHandler(std::move(targetHandler)), m_path(dirpath), m_since(modifiedSince), m_isDone(true), m_isBasedOnName(isBasedOnName), m_entryBuf(static_cast<struct dirent *>(::malloc(offsetof(struct dirent, d_name) + NAME_MAX + 1))) @@ -293,7 +293,7 @@ void FsVisitor::run(const DirPtr &dirptr, const FilePtr ¤tdir) auto ndirptr = openDir(fullpath); if (ndirptr == nullptr) { - WARN("Failed to open dir: " << fullpath); + WARN("Failed to open dir: " << fullpath << " with errno: " << errno); continue; } @@ -309,11 +309,12 @@ void FsVisitor::run(const DirPtr &dirptr, const FilePtr ¤tdir) } } - if (this->m_isBasedOnName && ncurrentdir->isInApp()) + if (this->m_isBasedOnName && ncurrentdir->isInApp()) { this->m_targetHandler(ncurrentdir); - - DEBUG("recurse dir : " << fullpath); - this->run(ndirptr, ncurrentdir); + } else { + DEBUG("recurse dir : " << fullpath); + this->run(ndirptr, ncurrentdir); + } } else if (result->d_type == DT_REG) { try { auto fileptr = File::createIfModified(fullpath, currentdir, this->m_since); @@ -336,11 +337,28 @@ void FsVisitor::run(const DirPtr &dirptr, const FilePtr ¤tdir) void FsVisitor::run() { auto dirptr = openDir(this->m_path); + + if (dirptr == nullptr) { + int err = errno; + WARN("Failed to open dir: " << this->m_path << " with errno: " << err << + ". let's retry."); + dirptr = openDir(this->m_path); + if (dirptr == nullptr) { + err = errno; + WARN("Failed to open dir again: " << this->m_path << " with errno: " << err << + ". let's skip."); + return; + } + } + auto currentdir = File::create(this->m_path, nullptr); INFO("Visiting files start from dir: " << this->m_path); - this->run(dirptr, currentdir); + if (this->m_isBasedOnName && currentdir->isInApp()) + this->m_targetHandler(currentdir); + else + this->run(dirptr, currentdir); } } // namespace Csr diff --git a/src/framework/service/server-service.cpp b/src/framework/service/server-service.cpp index 1ade03a..24bd1bf 100644 --- a/src/framework/service/server-service.cpp +++ b/src/framework/service/server-service.cpp @@ -33,6 +33,7 @@ #include "common/cs-detected.h" #include "common/wp-result.h" #include "common/exception.h" +#include "common/async-protocol.h" #include "service/exception.h" #include "service/access-control.h" #include "service/core-usage.h" @@ -53,9 +54,8 @@ std::string cidToString(const CommandId &cid) CID_TOSTRING(GET_DETECTED_LIST); CID_TOSTRING(GET_IGNORED); CID_TOSTRING(GET_IGNORED_LIST); - CID_TOSTRING(GET_SCANNABLE_FILES); - CID_TOSTRING(CANONICALIZE_PATHS); - CID_TOSTRING(SET_DIR_TIMESTAMP); + CID_TOSTRING(SCAN_DIRS_ASYNC); + CID_TOSTRING(SCAN_FILES_ASYNC); CID_TOSTRING(CANCEL_OPERATION); CID_TOSTRING(JUDGE_STATUS); @@ -82,7 +82,7 @@ inline CommandId extractCommandId(BinaryQueue &q) return id; } -} +} // namespace anonymous ServerService::ServerService() : Service(), m_workqueue(5) { @@ -155,18 +155,19 @@ RawBuffer ServerService::processCs(const ConnShPtr &conn, RawBuffer &data) return this->m_cslogic->scanFile(*cptr, filepath); } - case CommandId::GET_SCANNABLE_FILES: { + case CommandId::SCAN_FILES_ASYNC: { hasPermission(conn); - std::string dir; - q.Deserialize(dir); + CsContextShPtr cptr; + StrSet paths; + q.Deserialize(cptr, paths); auto fd = conn->getFd(); { std::lock_guard<std::mutex> l(this->m_cancelledMutex); this->m_isCancelled[fd] = false; - INFO("Turn off cancelled flag before scannable files start on fd: " << fd); + INFO("Turn off cancelled flag before start async. fd: " << fd); } Closer closer([this, fd]() { @@ -175,57 +176,52 @@ RawBuffer ServerService::processCs(const ConnShPtr &conn, RawBuffer &data) INFO("Erase cancelled flag in closer on fd: " << fd); }); - return this->m_cslogic->getScannableFiles(dir, [this, fd]() { + return this->m_cslogic->scanFilesAsync(conn, *cptr, paths, [this, fd]() { std::lock_guard<std::mutex> l(this->m_cancelledMutex); if (this->m_isCancelled.count(fd) == 1 && this->m_isCancelled[fd]) - ThrowExcInfo(-999, "operation cancelled on fd: " << fd); + ThrowExcInfo(ASYNC_EVENT_CANCEL, "operation cancelled on fd: " << fd); }); } - case CommandId::CANCEL_OPERATION: { - std::lock_guard<std::mutex> l(this->m_cancelledMutex); - auto fd = conn->getFd(); - if (this->m_isCancelled.count(fd) == 1) { - this->m_isCancelled[fd] = true; - INFO("Trun on cancelled flag of fd: " << fd); - } else { - WARN("Nothing to cancel on getting scannable list! fd: " << fd); - } - - return RawBuffer(); - } - - case CommandId::CANONICALIZE_PATHS: { + case CommandId::SCAN_DIRS_ASYNC: { hasPermission(conn); + CsContextShPtr cptr; StrSet paths; - q.Deserialize(paths); + q.Deserialize(cptr, paths); auto fd = conn->getFd(); { std::lock_guard<std::mutex> l(this->m_cancelledMutex); this->m_isCancelled[fd] = false; - INFO("Turn off cancelled flag before canonicalize paths start on fd: " << fd); + INFO("Turn off cancelled flag before start async. fd: " << fd); } Closer closer([this, fd]() { std::lock_guard<std::mutex> l(this->m_cancelledMutex); this->m_isCancelled.erase(fd); - INFO("Erase cancelled flag in closer of canonicalize paths on fd: " << fd); + INFO("Erase cancelled flag in closer on fd: " << fd); }); - return this->m_cslogic->canonicalizePaths(paths); + return this->m_cslogic->scanDirsAsync(conn, *cptr, paths, [this, fd]() { + std::lock_guard<std::mutex> l(this->m_cancelledMutex); + if (this->m_isCancelled.count(fd) == 1 && this->m_isCancelled[fd]) + ThrowExcInfo(ASYNC_EVENT_CANCEL, "operation cancelled on fd: " << fd); + }); } - case CommandId::SET_DIR_TIMESTAMP: { - hasPermission(conn); - - std::string dir; - int64_t ts64 = 0; - q.Deserialize(dir, ts64); + case CommandId::CANCEL_OPERATION: { + std::lock_guard<std::mutex> l(this->m_cancelledMutex); + auto fd = conn->getFd(); + if (this->m_isCancelled.count(fd) == 1) { + this->m_isCancelled[fd] = true; + INFO("Turn on cancelled flag of fd: " << fd); + } else { + WARN("Nothing to cancel... fd: " << fd); + } - return this->m_cslogic->setDirTimestamp(dir, static_cast<time_t>(ts64)); + return RawBuffer(); } case CommandId::JUDGE_STATUS: { @@ -432,6 +428,12 @@ void ServerService::onMessageProcess(const ConnShPtr &connection) if (!outbuf.empty()) connection->send(outbuf); + } catch (const Exception &e) { + if (e.error() == CSR_ERROR_SOCKET) + WARN("The connection is closed by the peer. Client might cancel async " + "scanning or crashed: " << e.what()); + else + throw; } catch (const std::exception &e) { ERROR("exception on workqueue task: " << e.what()); try { diff --git a/test/internals/test-db.cpp b/test/internals/test-db.cpp index 4ed8387..3b319e0 100644 --- a/test/internals/test-db.cpp +++ b/test/internals/test-db.cpp @@ -24,6 +24,7 @@ #include <iostream> #include <fstream> #include <string> +#include <chrono> #include <boost/test/unit_test.hpp> @@ -45,6 +46,25 @@ void checkSameMalware(const CsDetected &d, const Db::Row &r) ASSERT_IF(d.ts, r.ts); } +const char *appendIdxToStr(const char *str, int idx) +{ + return std::string(str + std::to_string(idx)).c_str(); +} + +using TimePoint = std::chrono::high_resolution_clock::time_point; + +TimePoint timeCheckStart() +{ + return std::chrono::high_resolution_clock::now(); +} + +void timeCheckEnd(TimePoint start) +{ + auto end = std::chrono::high_resolution_clock::now(); + std::chrono::duration<double> diff = end-start; + BOOST_MESSAGE("Elapsed time[" << diff.count() << "]. "); +} + } // namespace anonymous BOOST_AUTO_TEST_SUITE(DB) @@ -155,14 +175,14 @@ BOOST_AUTO_TEST_CASE(detected_malware_file) auto detectedList = db.getDetectedAllByNameOnDir("/opt", 0); ASSERT_IF(detectedList.empty(), true); - db.insertDetectedFile(malware1.targetName, malware1, initDataVersion); + db.insertDetectedFile(malware1, initDataVersion); detected = db.getDetectedAllByNameOnPath(malware1.targetName, 0); CHECK_IS_NOT_NULL(detected); checkSameMalware(malware1, *detected); ASSERT_IF(detected->dataVersion, initDataVersion); ASSERT_IF(detected->isIgnored, false); - db.insertDetectedFile(malware2.targetName, malware2, initDataVersion); + db.insertDetectedFile(malware2, initDataVersion); db.updateIgnoreFlag(malware2.targetName, true); detected = db.getDetectedAllByNameOnPath(malware2.targetName, 0); CHECK_IS_NOT_NULL(detected); @@ -170,7 +190,7 @@ BOOST_AUTO_TEST_CASE(detected_malware_file) ASSERT_IF(detected->dataVersion, initDataVersion); ASSERT_IF(detected->isIgnored, true); - db.insertDetectedFile(malware3.targetName, malware3, initDataVersion); + db.insertDetectedFile(malware3, initDataVersion); db.updateIgnoreFlag(malware3.targetName, true); detected = db.getDetectedAllByNameOnPath(malware3.targetName, 0); CHECK_IS_NOT_NULL(detected); @@ -200,7 +220,7 @@ BOOST_AUTO_TEST_CASE(detected_malware_file) ASSERT_IF(detected->isIgnored, true); // deleteDeprecatedDetectedMalwares test - db.insertDetectedFile(malware4.targetName, malware4, changedDataVersion); + db.insertDetectedFile(malware4, changedDataVersion); db.deleteDetectedDeprecated(3); detected = db.getDetectedAllByNameOnPath(malware4.targetName, 0); CHECK_IS_NOT_NULL(detected); @@ -221,4 +241,49 @@ BOOST_AUTO_TEST_CASE(detected_malware_file) EXCEPTION_GUARD_END } +BOOST_AUTO_TEST_CASE(transaction_time) +{ + EXCEPTION_GUARD_START + + Db::Manager db(TEST_DB_FILE, TEST_DB_SCRIPTS); + const int testSize = 500; + std::string dataVersion = "1.0.0"; + + // select test with vacant data + auto detectedList = db.getDetectedAllByNameOnDir("/opt", 0); + ASSERT_IF(detectedList.empty(), true); + + BOOST_MESSAGE("Start to time check about insert DB"); + auto start = timeCheckStart(); + db.transactionBegin(); + for(int i = 0; i < testSize; i++) { + CsDetected d; + d.targetName = appendIdxToStr("/opt/transmalware", i); + d.severity = CSR_CS_SEVERITY_LOW; + d.malwareName = appendIdxToStr("transmalware", i); + d.detailedUrl = appendIdxToStr("http://detailed.transmalware", i); + d.ts = 100; + + db.insertDetectedFile(d, dataVersion); + } + db.transactionEnd(); + timeCheckEnd(start); + + BOOST_MESSAGE("Start to time check about insert DB"); + auto start2 = timeCheckStart(); + for(int i = 0; i < testSize; i++) { + CsDetected d; + d.targetName = appendIdxToStr("/opt/testmalware", i); + d.severity = CSR_CS_SEVERITY_LOW; + d.malwareName = appendIdxToStr("testmalware", i); + d.detailedUrl = appendIdxToStr("http://detailed.malware", i); + d.ts = 100; + + db.insertDetectedFile(d, dataVersion); + } + timeCheckEnd(start2); + + EXCEPTION_GUARD_END +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/test/test-api-content-screening-async.cpp b/test/test-api-content-screening-async.cpp index c91769a..5e9b99c 100644 --- a/test/test-api-content-screening-async.cpp +++ b/test/test-api-content-screening-async.cpp @@ -957,8 +957,8 @@ BOOST_AUTO_TEST_CASE(get_malware_after_async) set_default_callback(context); const char *dirs[4] = { - "/opt/usr/media/", - "/opt/usr/apps/", + TEST_DIR_MEDIA(), + TEST_DIR_APPS(), "/tmp/", "/sdcard/" }; @@ -1173,4 +1173,33 @@ BOOST_AUTO_TEST_CASE(scan_async_multiple) EXCEPTION_GUARD_END } +BOOST_AUTO_TEST_CASE(scan_async_no_perm_dirs) +{ + EXCEPTION_GUARD_START + + std::string tmp_no_perm_media = std::string() + TEST_DIR_MEDIA() + "/tak"; + BOOST_MESSAGE("This TC needs special directory(" << tmp_no_perm_media << ") " + "which should be created manualy with special smack label to occur " + "smack-deny to open."); + + auto c = Test::Context<csr_cs_context_h>(); + auto context = c.get(); + + const char *dirs[3] = { + "/tmp", + tmp_no_perm_media.c_str(), + TEST_DIR_APPS() + }; + + set_default_callback(context); + + AsyncTestContext testCtx; + + ASSERT_SUCCESS(csr_cs_scan_dirs_async(context, dirs, 3, &testCtx)); + + ASSERT_CALLBACK(testCtx, -1, -1, 1, 0, 0); + + EXCEPTION_GUARD_END +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/test/test-api-content-screening.cpp b/test/test-api-content-screening.cpp index d2c29e2..240af4e 100644 --- a/test/test-api-content-screening.cpp +++ b/test/test-api-content-screening.cpp @@ -1296,6 +1296,7 @@ BOOST_AUTO_TEST_CASE(get_ignored_malwares_after_file_changed) // TODO: below test case needs response from UI. It'll be turned on as default after // write code of popup service stub +// For measuring TC coverage, eneable these test cases. #if 0 BOOST_AUTO_TEST_CASE(remove_failed_returns_detected_handle) { @@ -1317,6 +1318,35 @@ BOOST_AUTO_TEST_CASE(remove_failed_returns_detected_handle) EXCEPTION_GUARD_END } + +BOOST_AUTO_TEST_CASE(scan_file_wgt_dir_remove_app) +{ + EXCEPTION_GUARD_START + + auto c = Test::Context<csr_cs_context_h>(); + auto context = c.get(); + csr_cs_malware_h detected; + + Test::uninstall_app(TEST_WGT_PKG_ID); + ASSERT_INSTALL_APP(TEST_WGT_PATH, TEST_WGT_TYPE); + + ASSERT_SUCCESS(csr_cs_set_core_usage(context, CSR_CS_CORE_USAGE_DEFAULT)); + ASSERT_SUCCESS(csr_cs_scan_file(context, TEST_WGT_MAL_FILE(), &detected)); + + ASSERT_SUCCESS(csr_cs_set_core_usage(context, CSR_CS_CORE_USAGE_ALL)); + ASSERT_SUCCESS(csr_cs_scan_file(context, TEST_WGT_MAL_FILE(), &detected)); + + ASSERT_SUCCESS(csr_cs_set_core_usage(context, CSR_CS_CORE_USAGE_HALF)); + ASSERT_SUCCESS(csr_cs_scan_file(context, TEST_WGT_MAL_FILE(), &detected)); + + ASSERT_SUCCESS(csr_cs_set_core_usage(context, CSR_CS_CORE_USAGE_SINGLE)); + ASSERT_SUCCESS(csr_cs_scan_file(context, TEST_WGT_MAL_FILE(), &detected)); + + ASSERT_SUCCESS(csr_cs_set_ask_user(context, CSR_CS_ASK_USER_YES)); + ASSERT_SUCCESS(csr_cs_scan_file(context, TEST_WGT_MAL_FILE(), &detected)); + + EXCEPTION_GUARD_END +} #endif BOOST_AUTO_TEST_SUITE_END() diff --git a/test/test-api-engine-manager.cpp b/test/test-api-engine-manager.cpp index 7a4c217..fd9a6cc 100644 --- a/test/test-api-engine-manager.cpp +++ b/test/test-api-engine-manager.cpp @@ -78,6 +78,9 @@ BOOST_AUTO_TEST_CASE(fields_getters) ASSERT_SUCCESS(csr_engine_get_version(e.get(), &dataVersion.ptr)); ASSERT_IF(dataVersion.ptr, std::string("0.0.1")); + time_t updatedTime; + ASSERT_SUCCESS(csr_engine_get_latest_update_time(e.get(), &updatedTime)); + csr_activated_e activated; ASSERT_SUCCESS(csr_engine_get_activated(e.get(), &activated)); ASSERT_IF(activated, CSR_ACTIVATED); @@ -88,6 +91,8 @@ BOOST_AUTO_TEST_CASE(fields_getters) ASSERT_SUCCESS(csr_engine_get_state(e.get(), &state)); ASSERT_IF(state, CSR_STATE_ENABLE); + + EXCEPTION_GUARD_END } @@ -163,6 +168,9 @@ BOOST_AUTO_TEST_CASE(fields_getters) ASSERT_SUCCESS(csr_engine_get_version(e.get(), &dataVersion.ptr)); ASSERT_IF(dataVersion.ptr, std::string("0.0.1")); + time_t updatedTime; + ASSERT_SUCCESS(csr_engine_get_latest_update_time(e.get(), &updatedTime)); + csr_activated_e activated; ASSERT_SUCCESS(csr_engine_get_activated(e.get(), &activated)); ASSERT_IF(activated, CSR_ACTIVATED); |