C++11 封装支持公司可变参数的query
2021-08-22 本文已影响0人
FredricZhu
这里其实有两个封装，一个是HTTP支持发json和返回json，一个是读取json并构造测试用例。
总体来说都是针对网络的。
公司的这个Query是支持Web的，所以需要加上Chrome的两个UA头字段，否则直接用cpp-httplib请求基本失败。两个UA头字段是
image.png
另外POST请求返回的json会做gzip压缩，需要在cpp-httplib增加支持Zlib的宏，因为cpp-httplib会利用Zlib库对gzip格式进行解压缩，否则你直接看到gzip压缩格式的返回结果就是一团乱码。
而cpp-httplib也不会让这种情况发生。直接返回Error Read。
这就是C++语言的一个阴暗面了。错误后永远只看得到错误码，要看到更详细的error和异常堆栈基本不行，一般错误之后就是segement error了，段错误。需要自己逐行去debug看问题。对于多线程或者回调，有些函数断点进不去，最简单的方式就是直接使用std::cout输出。
拥有同样问题的另一个语言是Golang,但是不幸的是，Golang处理起来没那么方便。首先，Golang语言因为语言先天表达能力不足，所以代码写起来特别啰嗦，难看，正确逻辑被淹没在错误处理的茫茫大海中，还有不支持泛型导致大面积用于适配的垃圾代码出现，不利于查看逻辑。其次，Golang安装库都是只读的，不让改。
C++的好处就是大多数库都是头文件，你随便改。其次抽象能力强，代码短，逻辑清晰。基本保证一段代码能清晰执行自己的逻辑。不受所谓错误处理和类型适配的干扰。
增加宏的地方如图所示。
image.png
程序结构如下，
image.png
工程代码如下，
CMakeLists.txt
cmake_minimum_required(VERSION 2.6)
project(http_util_test)
add_definitions(-std=c++14)
add_definitions(-g)
find_package(ZLIB)
find_package(Boost REQUIRED COMPONENTS
system
filesystem
serialization
program_options
thread
)
include_directories(${Boost_INCLUDE_DIRS} /usr/local/include ${CMAKE_CURRENT_SOURCE_DIR}/../../)
LINK_DIRECTORIES(/usr/local/lib)
file( GLOB APP_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp ${CMAKE_CURRENT_SOURCE_DIR}/*.h
${CMAKE_CURRENT_SOURCE_DIR}/../impl/*.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../../http/impl/*.cpp ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp)
foreach( sourcefile ${APP_SOURCES} )
file(RELATIVE_PATH filename ${CMAKE_CURRENT_SOURCE_DIR} ${sourcefile})
string(FIND "${filename}" "test.cpp" "TEMP")
if( NOT "${TEMP}" STREQUAL "-1" )
string(REPLACE ".cpp" "" file ${filename})
add_executable(${file} ${APP_SOURCES})
target_link_libraries(${file} ${Boost_LIBRARIES} ZLIB::ZLIB)
target_link_libraries(${file} ssl crypto libgtest.a libgtest_main.a pystring libgmock.a pthread)
endif()
endforeach( sourcefile ${APP_SOURCES})
config.h
#ifndef _FREDRIC_CONFIG_H_
#define _FREDRIC_CONFIG_H_
#include "query_ele.h"
#include <map>
#include <string>
#include <vector>
// HTTP服务器Host地址
const std::string Host = "tai.appannie.com";
// 所有新增加的GP支持的国家
const std::vector<std::string> GPCountries{
"IR", "BD", "BH", "EC", "GT", "KG", "LK", "MD", "RS", "TT",
"AM", "DZ", "BA", "GH", "BO", "BY", "LU", "DO", "HN", "KH",
"IQ", "JM", "JO", "MO", "SI", "UZ", "MK", "NP", "MA", "PY",
"SV", "OM", "PA", "TZ", "QA", "TN", "UY", "VE", "YE"};
// 通用的 HTTP头配置
std::map<std::string, std::string> headers{
{"Authorization", "Bearer {YOUR_API_KEY›}"},
{"pragma", "no-cache"},
{"cache-control", "no-cache"},
{"X-AA-DMS-NAMESPACE", "taiaaauthorisationservice939"},
{"sec-ch-ua",
R"(Chromium";v="92", " Not A;Brand";v="99", "Microsoft Edge";v="92")"},
{"user-agent",
R"(Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36 Edg/92.0.902.73)"},
{"disable-udw-reader-sql-cache", "1"}};
// Query的 json文件存放的位置
const std::string QueryDataPath = "../../data/";
#endif
test_config.h
#ifndef _FREDRIC_TEST_CONFIG_H_
#define _FREDRIC_TEST_CONFIG_H_
#include <map>
#include <string>
#include "query_ele.h"
const std::string StoreProductRankPaidCase = "StoreProductRankPaidCase";
const std::string TopCompanyOverviewTableCase = "TopCompanyOverviewTableCase";
const std::string DownloadChannelTableCase = "DownloadChannelTableCase";
std::map<const std::string, QueryEle> TestEles{
{StoreProductRankPaidCase,
{"/ajax/v2/"
"query?query_identifier=table_change%28top_apps%24gp_overview_paid%29",
"store_product_rank_paid"}},
{TopCompanyOverviewTableCase,
{"/ajax/v2/"
"query?query_identifier=table_change%28top_company_overview_table_v0_1%"
"29",
"top_company_overview_table"}},
{DownloadChannelTableCase,
{"/ajax/v2/"
"query?query_identifier=table_change%28download_channel_table%29",
"download_channel_table"}}};
#endif
query_ele.h
#ifndef _FREDRIC_QUERY_ELE_H_
#define _FREDRIC_QUERY_ELE_H_
#include <string>
struct QueryEle {
std::string path{};
std::string query_body_name{};
};
#endif
check_gp_countries.h
#ifndef _FREDRIC_CHECK_GP_COUNTRIES_H_
#define _FREDRIC_CHECK_GP_COUNTRIES_H_
#include "query_ele.h"
bool send_request_query(const QueryEle& ele);
#endif
check_gp_countries.cpp
#include "check_gp_countrires/check_gp_countries.h"
#include "http/http_util.h"
#include "pystring/pystring.h"
#include "check_gp_countrires/config.h"
#include "json/json.hpp"
#include <fstream>
#include <sstream>
using json = nlohmann::json;
bool send_request_query(const QueryEle& ele) {
std::string query_path = ele.path;
std::string query_body_path = QueryDataPath + ele.query_body_name + ".json";
std::ifstream query_body_file(query_body_path);
std::stringstream buffer;
buffer << query_body_file.rdbuf();
std::string query_body(buffer.str());
std::vector<std::string> failed_countrires = {};
for(auto&& country :GPCountries) {
std::cout << "Country: [" << country << "]" << std::endl;
std::string body_str = pystring::replace(query_body, "{COUNTRY_CODE}", country);
std::string result{};
bool get_str_res = HttpUtil::post_and_get_str(Host, query_path, headers, body_str, result);
if(!get_str_res) {
failed_countrires.emplace_back(std::move(country));
}
auto res = json::parse(result);
auto facet_len = res["data"]["facets"].size();
if(facet_len == 0) {
std::cout << "Country: [" << country << "]" << " has no data in DB!" << std::endl;
}
}
return failed_countrires.size() == 0;
}
check_gp_countries_test.cpp
#include "check_gp_countrires/test_config.h"
#include "check_gp_countrires/check_gp_countries.h"
#include <gtest/gtest.h>
bool run_a_test_case(const std::string& case_name) {
std::cout << "Test Case: [" << case_name << "] Started !" << std::endl;
auto test_case = TestEles[case_name];
auto res = send_request_query(test_case);
std::cout << "Test Case: [" << case_name << "] End !" << std::endl;
return res;
}
GTEST_TEST(CheckGPCountriesTests, StoreProductRankPaidCase) {
bool test_result = run_a_test_case(StoreProductRankPaidCase);
ASSERT_EQ(true, test_result);
}
GTEST_TEST(CheckGPCountriesTests, TopCompanyOverviewTableCase) {
bool test_result = run_a_test_case(TopCompanyOverviewTableCase);
ASSERT_EQ(true, test_result);
}
GTEST_TEST(CheckGPCountriesTests, DownloadChannelTableCase) {
bool test_result = run_a_test_case(DownloadChannelTableCase);
ASSERT_EQ(true, test_result);
}
data下面都是query的json文件。
image.png
http_util.h
#ifndef _HTTP_UTIL_H_
#define _HTTP_UTIL_H_
#define CPPHTTPLIB_OPENSSL_SUPPORT
#define CPPHTTPLIB_ZLIB_SUPPORT
#include "http/httplib.h"
#include <string>
const int ConnectionTimeout = 30;
class HttpUtil {
public:
/**
* HttpUtil get method
*
* @param: url the url to be used to get a web page from remote server
* @param: path the path to be used to get a web page from remote server
* @param: result_name the download result file path
*/
static bool get(std::string url, std::string path, std::string result_name);
/**
* HttpUtil get_file method
*
* @param: host the host to be used to get an item from remote server
* @param: path the path to be used to get an item from remote server
* @param: result_name the download result file path
*/
static bool get_file(std::string host, std::string path, std::string result_name);
static bool get_str(std::string host, std::string path, const std::map<std::string, std::string> & headers, std::string &result_string);
static bool post_and_get_str(std::string host, std::string path, const std::map<std::string, std::string> & headers, const std::string& body,std::string &result_string);
};
#endif
http_util.cpp
#include "http/http_util.h"
#include <iostream>
#include <fstream>
bool HttpUtil::get(std::string url, std::string path, std::string result_name) {
try {
httplib::Client cli {url};
cli.set_connection_timeout(ConnectionTimeout);
auto res = cli.Get(path.c_str());
if(res->status != 200) {
std::cerr << "Get [" << url << path << "] failed" << std::endl;
std::cerr << "Status code : [" << res->status << "]" << "\n" << "Result body : [" << res->body << "]"
<< std::endl;
return false;
}
std::ofstream os {result_name, std::ios_base::out | std::ios_base::binary};
os << res->body;
} catch(const std::exception & e) {
std::cerr << "Exception: " << e.what() << std::endl;
return false;
}
return true;
}
bool HttpUtil::get_file(std::string host, std::string path, std::string result_name) {
try {
#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
auto port = 443;
httplib::SSLClient cli(host, port);
#else
auto port = 80;
httplib::Client cli(host, port);
#endif
cli.set_connection_timeout(ConnectionTimeout);
std::ofstream os {result_name};
auto res = cli.Get(path.c_str(),
[&](const char *data, size_t data_length) {
os << std::string(data, data_length);
return true;
});
if(!res || res->status != 200) {
return false;
}
} catch(const std::exception & e) {
std::cerr << "Exception: " << e.what() << std::endl;
return false;
}
return true;
}
bool HttpUtil::get_str(std::string host, std::string path, const std::map<std::string, std::string> & headers, std::string &result_string) {
try {
#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
auto port = 443;
httplib::SSLClient cli(host, port);
#else
auto port = 80;
httplib::Client cli(host, port);
#endif
cli.set_connection_timeout(ConnectionTimeout);
httplib::Headers headers_ {};
for(auto&& item: headers) {
headers_.insert(item);
}
auto res = cli.Get(path.c_str(), headers_);
result_string = res->body;
return true;
} catch(const std::exception & e) {
std::cerr << "Exception: " << e.what() << std::endl;
return false;
}
}
bool HttpUtil::post_and_get_str(std::string host, std::string path, const std::map<std::string, std::string> & headers, const std::string& body, std::string &result_string){
try {
#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
auto port = 443;
httplib::SSLClient cli(host, port);
#else
auto port = 80;
httplib::Client cli(host, port);
#endif
cli.set_connection_timeout(ConnectionTimeout);
httplib::Headers headers_ {};
for(auto&& item: headers) {
headers_.insert(item);
}
auto res = cli.Post(path.c_str(), headers_, body.c_str(), body.size(), "application/json");
result_string = res->body;
return true;
} catch(const std::exception & e) {
std::cerr << "Exception: " << e.what() << std::endl;
return false;
}
}
程序输出如下，
image.png
