From da875602b3c0534f687416fe736759cba3fc4ea8 Mon Sep 17 00:00:00 2001 From: liuxianliang Date: Wed, 9 Jun 2021 14:11:23 +0800 Subject: [PATCH] [add] shard download function [remove] web get position function --- README.md | 2 +- README_ZH.md | 2 +- docs/api.md | 8 +- docs/samples.md | 70 ++++++++--- docs/user-guide.md | 19 ++- inc/webclient.h | 6 +- samples/webclient_shard_download_sample.c | 137 ++++++++++++++++++++++ src/webclient.c | 116 +++++++++++++----- 8 files changed, 296 insertions(+), 64 deletions(-) create mode 100644 samples/webclient_shard_download_sample.c diff --git a/README.md b/README.md index 407740b..5d31a1e 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ RT-Thread online packages IoT-internet of things ---> [*] WebClient: A HTTP/HTTPS Client for RT-Thread [ ] Enable debug log output - [ ] Enable webclient GET/POST samples + [ ] Enable webclient GET/POST/SHARD samples [ ] Enable file download feature support Select TLS mode (Not support) ---> (x) Not support diff --git a/README_ZH.md b/README_ZH.md index c6b6531..70199e6 100644 --- a/README_ZH.md +++ b/README_ZH.md @@ -59,7 +59,7 @@ WebClient 软件包遵循 Apache-2.0 许可，详见 LICENSE 文件。 RT-Thread online packages IoT - internet of things ---> [*] WebClient: A HTTP/HTTPS Client for RT-Thread - [ ] Enable webclient GET/POST samples + [ ] Enable webclient GET/POST/SHARD samples Select TLS mode (Not support) ---> (x) Not support ( ) SAL TLS support diff --git a/docs/api.md b/docs/api.md index f722a50..ac98d88 100644 --- a/docs/api.md +++ b/docs/api.md @@ -46,19 +46,19 @@ int webclient_get(struct webclient_session *session, const char *URI); |`>0` | HTTP 响应状态码 | |<0 | 发送请求失败 | -## 发送获取部分数据的 GET 请求 +## 获取指定数据大小的 HEAD / GET 请求 ```c -int webclient_get_position(struct webclient_session *session, const char *URI, int position); +int webclient_shard_position_function(struct webclient_session *session, const char *URI, int size); ``` -发送带有 Range 头信息的 HTTP GET 请求命令，多用于完成断点续传功能。 +发送带有 Range 头信息的 HTTP GET/HEAD 请求命令，多用于断点续传 / 分片下载功能。 | 参数 | 描述 | |:------------------|:-----------------------------------| |session | 当前连接会话结构体指针 | |URI | 连接的 HTTP 服务器地址 | -|position | 数据偏移量 | +|size | 设定的接收空间 | | **返回** | **描述** | |`>0` | HTTP 响应状态码 | |<0 | 发送请求失败 | diff --git a/docs/samples.md b/docs/samples.md index 5ac0dab..a3563e5 100644 --- a/docs/samples.md +++ b/docs/samples.md @@ -1,13 +1,14 @@ # 示例程序 -WebClient 软件包提供两个 HTTP Client 示例程序, 分别用于演示软件包支持的 GET 和 POST 功能，完成数据的上传与下载。 +WebClient 软件包提供三个 HTTP Client 示例程序, 分别用于演示软件包支持的 GET 和 POST 功能，完成数据的上传与下载；以及一个完整的分片下载的功能。 **示例文件** -| 示例程序路径 | 说明 | -| ---- | ---- | -| samples/webclient_get_sample.c | GET 请求测试例程 | -| samples/webclient_post_sample.c | POST 请求测试例程 | +| 示例程序路径 | 说明 | +| ---- | ---- | +| samples/webclient_get_sample.c | GET 请求测试例程 | +| samples/webclient_post_sample.c | POST 请求测试例程 | +| samples/webclient_shard_download_sample.c | 分片下载测试例程 | ## 准备工作 @@ -17,14 +18,14 @@ WebClient 软件包提供两个 HTTP Client 示例程序, 分别用于演示软 打开 RT-Thread 提供的 ENV 工具，使用 **menuconfig** 配置软件包。 - 启用 WebClient 软件包，并配置使能测试例程（Enable webclient GET/POST samples），如下所示： + 启用 WebClient 软件包，并配置使能测试例程（Enable webclient GET/POST/SHARD samples），如下所示： ```shell RT-Thread online packages IoT - internet of things ---> - [*] WebClient: A HTTP/HTTPS Client for RT-Thread - [ ] Enable debug log output - [*] Enable webclient GET/POST samples # 开启 WebClient 测试例程 + [*] WebClient: A HTTP/HTTPS Client for RT-Thread + [ ] Enable debug log output + [*] Enable webclient GET/POST/SHARD samples # 开启 WebClient 测试例程 [ ] Enable file download feature support Select TLS mode (Not support) ---> Version (latest) ---> # 开启使用最新版本软件包 @@ -34,7 +35,7 @@ RT-Thread online packages - 编译下载 -## 启动例程 +## 启动例程 本例程使用的测试网站是 RT-Thread 系统的官方网站。GET 请求示例可以从网站中获取并打印显示文件内容；POST 请求示例可以上传数据到测试网站，测试网站会响应相同的数据。 @@ -55,13 +56,13 @@ GET 请求示例使用方式有如下两种： - 在 MSH 中使用命令 `web_get_test` 执行 GET 请求示例程序，可以获取并打印显示默认网址下载的文件信息;在 MSH 中使用命令 `web_get_test -s` 执行 POST 请求示例程序，使用简化接口（webclient_request 接口）发送 GET请求，适用于简短数据的收发。如下图 LOG 显示： ```c -msh />web_get_test -webclient get response data: +msh />web_get_test +webclient get response data: RT-Thread is an open source IoT operating system from China, which has strong scalability: from a tiny kernel running on a tiny core, for example ARM Cortex-M0, or Cortex-M3/4/7, to a rich feature system running on MIPS32, ARM Cortex-A8, ARM Cortex-A9 DualCore etc. msh />web_get_test -s webclient send get request by simplify request interface. -webclient get response data: +webclient get response data: RT-Thread is an open source IoT operating system from China, which has strong scalability: from a tiny kernel running on a tiny core, for example ARM Cortex-M0, or Cortex-M3/4/7, to a rich feature system running on MIPS32, ARM Cortex-A8, ARM Cortex-A9 DualCore etc. msh /> @@ -88,12 +89,49 @@ POST 请求示例使用方式有如下两种： msh />web_post_test webclient post response data : RT-Thread is an open source IoT operating system from China! -msh /> +msh /> msh />web_post_test -s webclient send post request by simplify request interface. -webclient post response data: +webclient post response data: RT-Thread is an open source IoT operating system from China! msh /> ``` -- 在 MSH 中使用命令 `web_post_test [URI]` 或者 `web_post_test -s [URI]` 格式命令执行 POST 请求示例程序，其中 URI 为用户自定义的支持 POST 请求的地址。 \ No newline at end of file +- 在 MSH 中使用命令 `web_post_test [URI]` 或者 `web_post_test -s [URI]` 格式命令执行 POST 请求示例程序，其中 URI 为用户自定义的支持 POST 请求的地址。 + +### 分片下载示例 + +分片下载示例流程： + +- 创建 client 会话结构体 +- client 发送 HEAD 请求 header 数据 +- server 响应 header 数据 +- client 发送 GET 请求 header 数据，包含 Range 字段 +- server 响应 header 数据和指定长度的 body 数据 +- 循环发送请求直到接收完成所有数据 +- 分片下载测试完成/失败 + +GET 请求示例使用方式有如下两种： + +- 在 MSH 中使用命令 `web_get_test -l [size]` 执行分片下载示例程序；指定允许接收的最大 body 数据长度，可以获取并打印显示默认网址下载的文件信息; +```c +msh >web_shard_test -l 115 +Receive, len[0115]: +0000 - 0059: RT-Thread is an open source IoT operating system from China, +0060 - 0114: which has strong scalability: from a tiny kernel runni +Total: [0115]Bytes + +Receive, len[0115]: +0000 - 0059: ng on a tiny core, for example ARM Cortex-M0, or Cortex-M3/4 +0060 - 0114: /7, to a rich feature system running on MIPS32, ARM Cor +Total: [0115]Bytes + +Receive, len[0037]: +0000 - 0036: tex-A8, ARM Cortex-A9 DualCore etc. + +Total: [0037]Bytes +msh /> +``` +分多次下载得到的数据，与通用的 GET 请求获取的数据完全一致，分片下载功能正常。 + +- 在 MSH 中使用命令 `web_get_test -u [URI]` 格式命令执行 GET 请求示例程序，其中 URI 为用户自定义的支持 GET 请求的地址。 \ No newline at end of file diff --git a/docs/user-guide.md b/docs/user-guide.md index f08de01..f37b76d 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -67,7 +67,7 @@ struct webclient_session int content_length; //当前接收数据长度（非 chunk 模式） size_t content_remainder; //当前剩余接收数据长度 - + rt_bool_t is_tls; //当前连接是否是 HTTPS 连接 #ifdef WEBCLIENT_USING_MBED_TLS MbedTLSSession *tls_session; // HTTPS 协议相关会话结构体 @@ -201,7 +201,7 @@ while(1) { webclient_read(session, buffer, bfsz); ... -} +} webclient_close(session); ``` @@ -224,28 +224,27 @@ while(1) { webclient_read(session, buffer, bfsz); ... -} +} webclient_close(session); ``` -- 发送获取部分数据的 GET 请求（多用于断点续传） +- 发送获取部分数据的 GET 请求（多用于断点续传/分片下载） ```c struct webclient_session *session = NULL; session = webclient_create(1024); -if(webclient_get_position(URI, 100) != 206) -{ - LOG_E("error!"); -} +webclient_connect(session, URI); +webclient_header_fields_add(session, "Range: bytes=%d-%d\r\n", 0, 99); +webclient_send_header(session, WEBCLIENT_GET); while(1) { webclient_read(session, buffer, bfsz); ... -} +} webclient_close(session)； ``` @@ -313,7 +312,7 @@ while(1) { webclient_write(session, post_data, 1024); ... -} +} if( webclient_handle_response(session) != 200) { diff --git a/inc/webclient.h b/inc/webclient.h index 45977a2..0477be1 100644 --- a/inc/webclient.h +++ b/inc/webclient.h @@ -71,6 +71,7 @@ enum WEBCLIENT_METHOD WEBCLIENT_USER_METHOD, WEBCLIENT_GET, WEBCLIENT_POST, + WEBCLIENT_HEAD }; struct webclient_header @@ -95,6 +96,7 @@ struct webclient_session int content_length; size_t content_remainder; /* remainder of content length */ + int (*handle_function)(char *buffer, int size); /* handle function */ rt_bool_t is_tls; /* HTTPS connect */ #ifdef WEBCLIENT_USING_MBED_TLS @@ -107,7 +109,9 @@ struct webclient_session *webclient_session_create(size_t header_sz); /* send HTTP GET request */ int webclient_get(struct webclient_session *session, const char *URI); -int webclient_get_position(struct webclient_session *session, const char *URI, int position); +int webclient_shard_position_function(struct webclient_session *session, const char *URI, int size); + +char *webclient_register_shard_position_function(struct webclient_session *session, int (*handle_function)(char *buffer, int size)); /* send HTTP POST request */ int webclient_post(struct webclient_session *session, const char *URI, const void *post_data, size_t data_len); diff --git a/samples/webclient_shard_download_sample.c b/samples/webclient_shard_download_sample.c new file mode 100644 index 0000000..bceff39 --- /dev/null +++ b/samples/webclient_shard_download_sample.c @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2006-2018, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2021-06-03 xiangxistu the first version + */ + +#include +#include +#include "stdlib.h" + +#define GET_LOCAL_URI "http://www.rt-thread.com/service/rt-thread.txt" +#define CHARACTER_LENGTH 60 + +/* handle function, you can store data and so on */ +static int shard_download_handle(char *buffer, int length) +{ + int outindex, inindex = 0; + int boundary; + + /* print the receive data */ + rt_kprintf("\nReceive, len[%04d]:\n", length); + for (outindex = 0; outindex < length; outindex = outindex + inindex) + { + char print_buffer[CHARACTER_LENGTH + 1] = {0}; + char *point = RT_NULL; + point = print_buffer; + + if(length - outindex > CHARACTER_LENGTH) + { + boundary = CHARACTER_LENGTH; + } + else + { + boundary = length - outindex; + } + + for (inindex = 0; inindex < boundary; inindex++) + { + *point++ = buffer[outindex + inindex]; + } + *point = 0; + rt_kprintf("%04d - %04d: %s\n", outindex, outindex + boundary - 1, print_buffer); + } + rt_kprintf("Total: [%04d]Bytes\n", length); + + /* release this buffer if we have handled data */ + web_free(buffer); + + return RT_EOK; +} + + +int webclient_shard_download_test(int argc, char **argv) +{ + struct webclient_session* session = RT_NULL; + rt_err_t result = RT_EOK; + char *uri = RT_NULL; + int size = 0; + int usage_flag = 0; + + + if (argc == 1) + { + uri = web_strdup(GET_LOCAL_URI); + } + else + { + int index; + for(index = 1; index < argc; index = index + 2) + { + if(rt_strstr(argv[index], "-u")) + { + uri = web_strdup(argv[index + 1]); + } + else if(rt_strstr(argv[index], "-l")) + { + size = atoi(argv[index + 1]); + } + else + { + usage_flag = 1; + break; + } + } + } + + if(usage_flag) + { + rt_kprintf("web_shard_test -u [URI] - webclient HEAD and GET request test.\n"); + rt_kprintf("web_shard_test -l [SIZE] - the length of receive buffer.\n"); + return -RT_ERROR; + } + + if(uri == RT_NULL) + { + uri = web_strdup(GET_LOCAL_URI); + } + + /* sometime, the header bufsz can set more smaller */ + session = webclient_session_create(WEBCLIENT_HEADER_BUFSZ / 4); + if (session == RT_NULL) + { + result = -RT_ENOMEM; + goto __exit; + } + + /* register the handle function, you can handle data in the function */ + webclient_register_shard_position_function(session, shard_download_handle); + + /* the "memory size" that you can provide in the project and uri */ + webclient_shard_position_function(session, uri, size); + + /* clear the handle function */ + webclient_register_shard_position_function(session, RT_NULL); + +__exit: + if (uri) + { + web_free(uri); + } + + if (session) + { + webclient_close(session); + } + + return result; +} + +#ifdef FINSH_USING_MSH +#include +MSH_CMD_EXPORT_ALIAS(webclient_shard_download_test, web_shard_test, webclient head and get request test); +#endif /* FINSH_USING_MSH */ diff --git a/src/webclient.c b/src/webclient.c index f52eede..6747bd6 100644 --- a/src/webclient.c +++ b/src/webclient.c @@ -12,6 +12,7 @@ * 2018-01-04 aozima add ipv6 address support. * 2018-07-26 chenyong modify log information * 2018-08-07 chenyong modify header processing + * 2021-06-09 xiangxistu add shard download function */ #include @@ -626,7 +627,7 @@ static int webclient_send_header(struct webclient_session *session, int method) header = session->header->buffer; - if (session->header->length == 0) + if (session->header->length == 0 && method <= WEBCLIENT_GET) { /* use default header data */ if (webclient_header_fields_add(session, "GET %s HTTP/1.1\r\n", session->req_url) < 0) @@ -663,6 +664,9 @@ static int webclient_send_header(struct webclient_session *session, int method) else if (method == WEBCLIENT_POST) length = rt_snprintf(session->header->buffer, session->header->size, "POST %s HTTP/1.1\r\n%s", session->req_url ? session->req_url : "/", header_buffer); + else if (method == WEBCLIENT_HEAD) + length = rt_snprintf(session->header->buffer, session->header->size, "HEAD %s HTTP/1.1\r\n%s", + session->req_url ? session->req_url : "/", header_buffer); session->header->length = length; web_free(header_buffer); @@ -963,19 +967,36 @@ int webclient_get(struct webclient_session *session, const char *URI) } /** - * http breakpoint resume. + * register a handle function for http breakpoint resume and shard download. + * + * @param function + * + * @return the pointer + */ +char *webclient_register_shard_position_function(struct webclient_session *session, int (*handle_function)(char *buffer, int size)) +{ + session->handle_function = handle_function; + + return (char *)session->handle_function; +} + +/** + * http breakpoint resume and shard download. * * @param session webclient session * @param URI input server URI address - * @param position last downloaded position + * @param the buffer size that you alloc * * @return <0: send GET request failed * >0: response http status code */ -int webclient_get_position(struct webclient_session *session, const char *URI, int position) +int webclient_shard_position_function(struct webclient_session *session, const char *URI, int size) { int rc = WEBCLIENT_OK; int resp_status = 0; + int real_total_len = 0; + int start_position, end_position = 0; + char *buffer = RT_NULL; RT_ASSERT(session); RT_ASSERT(URI); @@ -986,50 +1007,83 @@ int webclient_get_position(struct webclient_session *session, const char *URI, i return rc; } - /* splice header*/ - if (webclient_header_fields_add(session, "Range: bytes=%d-\r\n", position) <= 0) - { - rc = -WEBCLIENT_ERROR; - return rc; - } - - rc = webclient_send_header(session, WEBCLIENT_GET); + rc = webclient_send_header(session, WEBCLIENT_HEAD); if (rc != WEBCLIENT_OK) { return rc; } - /* handle the response header of webclient server */ + /* handle the response header of webclient server by HEAD request */ resp_status = webclient_handle_response(session); + if(resp_status >= 0) + { + real_total_len = webclient_content_length_get(session); + LOG_D("The length[%04d] of real data of URI.", real_total_len); + } - LOG_D("get position handle response(%d).", resp_status); + /* clean header buffer and size */ + rt_memset(session->header->buffer, 0x00, session->header->size); + session->header->length = 0; - if (resp_status > 0) + for(start_position = end_position; end_position < real_total_len; start_position = end_position + 1) { - const char *location = webclient_header_fields_get(session, "Location"); + RT_ASSERT(start_position < real_total_len); + int data_len = 0; - /* relocation */ - if ((resp_status == 302 || resp_status == 301) && location) + end_position = start_position + size - 1; + if(end_position >= real_total_len) { - char *new_url; + end_position = real_total_len; + } - new_url = web_strdup(location); - if (new_url == RT_NULL) + /* splice header and send header */ + LOG_I("Range: [%04d -> %04d]", start_position, end_position); + webclient_header_fields_add(session, "Range: bytes=%d-%d\r\n", start_position, end_position); + rc = webclient_send_header(session, WEBCLIENT_GET); + if (rc != WEBCLIENT_OK) + { + return rc; + } + + /* handle the response header of webclient server */ + resp_status = webclient_handle_response(session); + + LOG_D("get position handle response(%d).", resp_status); + if (resp_status > 0) + { + const char *location = webclient_header_fields_get(session, "Location"); + + /* relocation */ + if ((resp_status == 302 || resp_status == 301) && location) { - return -WEBCLIENT_NOMEM; - } + char *new_url; - /* clean webclient session */ - webclient_clean(session); - /* clean webclient session header */ - session->header->length = 0; - rt_memset(session->header->buffer, 0, session->header->size); + new_url = web_strdup(location); + if (new_url == RT_NULL) + { + return -WEBCLIENT_NOMEM; + } - rc = webclient_get_position(session, new_url, position); + /* clean webclient session */ + webclient_clean(session); + /* clean webclient session header */ + session->header->length = 0; + rt_memset(session->header->buffer, 0, session->header->size); - web_free(new_url); - return rc; + rc = webclient_shard_position_function(session, new_url, size); + + web_free(new_url); + return rc; + } } + + /* receive the incoming data */ + data_len = webclient_response(session, &buffer, RT_NULL); + session->handle_function(buffer, data_len); + + /* clean header buffer and size */ + rt_memset(session->header->buffer, 0x00, session->header->size); + session->header->length = 0; } return resp_status;