[add] shard download function

[remove] web get position function
master
liuxianliang 4 years ago
parent 0a516a9742
commit da875602b3

@ -60,7 +60,7 @@ RT-Thread online packages
IoT-internet of things --->
[*] WebClient: A HTTP/HTTPS Client for RT-Thread
[ ] Enable debug log output
[ ] Enable webclient GET/POST samples
[ ] Enable webclient GET/POST/SHARD samples
[ ] Enable file download feature support
Select TLS mode (Not support) --->
(x) Not support

@ -59,7 +59,7 @@ WebClient 软件包遵循 Apache-2.0 许可,详见 LICENSE 文件。
RT-Thread online packages
IoT - internet of things --->
[*] WebClient: A HTTP/HTTPS Client for RT-Thread
[ ] Enable webclient GET/POST samples
[ ] Enable webclient GET/POST/SHARD samples
Select TLS mode (Not support) --->
(x) Not support
( ) SAL TLS support

@ -46,19 +46,19 @@ int webclient_get(struct webclient_session *session, const char *URI);
|`>0` | HTTP 响应状态码 |
|<0 | |
## 发送获取部分数据的 GET 请求
## 获取指定数据大小的 HEAD / GET 请求
```c
int webclient_get_position(struct webclient_session *session, const char *URI, int position);
int webclient_shard_position_function(struct webclient_session *session, const char *URI, int size);
```
发送带有 Range 头信息的 HTTP GET 请求命令,多用于完成断点续传功能。
发送带有 Range 头信息的 HTTP GET/HEAD 请求命令,多用于断点续传 / 分片下载功能。
| 参数 | 描述 |
|:------------------|:-----------------------------------|
|session | 当前连接会话结构体指针 |
|URI | 连接的 HTTP 服务器地址 |
|position | 数据偏移量 |
|size | 设定的接收空间 |
| **返回** | **描述** |
|`>0` | HTTP 响应状态码 |
|<0 | |

@ -1,6 +1,6 @@
# 示例程序
WebClient 软件包提供个 HTTP Client 示例程序, 分别用于演示软件包支持的 GET 和 POST 功能,完成数据的上传与下载。
WebClient 软件包提供个 HTTP Client 示例程序, 分别用于演示软件包支持的 GET 和 POST 功能,完成数据的上传与下载;以及一个完整的分片下载的功能
**示例文件**
@ -8,6 +8,7 @@ WebClient 软件包提供两个 HTTP Client 示例程序, 分别用于演示软
| ---- | ---- |
| samples/webclient_get_sample.c | GET 请求测试例程 |
| samples/webclient_post_sample.c | POST 请求测试例程 |
| samples/webclient_shard_download_sample.c | 分片下载测试例程 |
## 准备工作
@ -17,14 +18,14 @@ WebClient 软件包提供两个 HTTP Client 示例程序, 分别用于演示软
打开 RT-Thread 提供的 ENV 工具,使用 **menuconfig** 配置软件包。
启用 WebClient 软件包并配置使能测试例程Enable webclient GET/POST samples如下所示
启用 WebClient 软件包并配置使能测试例程Enable webclient GET/POST/SHARD samples如下所示
```shell
RT-Thread online packages
IoT - internet of things --->
[*] WebClient: A HTTP/HTTPS Client for RT-Thread
[ ] Enable debug log output
[*] Enable webclient GET/POST samples # 开启 WebClient 测试例程
[*] Enable webclient GET/POST/SHARD samples # 开启 WebClient 测试例程
[ ] Enable file download feature support
Select TLS mode (Not support) --->
Version (latest) ---> # 开启使用最新版本软件包
@ -97,3 +98,40 @@ msh />
```
- 在 MSH 中使用命令 `web_post_test [URI]` 或者 `web_post_test -s [URI]` 格式命令执行 POST 请求示例程序,其中 URI 为用户自定义的支持 POST 请求的地址。
### 分片下载示例
分片下载示例流程:
- 创建 client 会话结构体
- client 发送 HEAD 请求 header 数据
- server 响应 header 数据
- client 发送 GET 请求 header 数据,包含 Range 字段
- server 响应 header 数据和指定长度的 body 数据
- 循环发送请求直到接收完成所有数据
- 分片下载测试完成/失败
GET 请求示例使用方式有如下两种:
- 在 MSH 中使用命令 `web_get_test -l [size]` 执行分片下载示例程序;指定允许接收的最大 body 数据长度,可以获取并打印显示默认网址下载的文件信息;
```c
msh >web_shard_test -l 115
Receive, len[0115]:
0000 - 0059: RT-Thread is an open source IoT operating system from China,
0060 - 0114: which has strong scalability: from a tiny kernel runni
Total: [0115]Bytes
Receive, len[0115]:
0000 - 0059: ng on a tiny core, for example ARM Cortex-M0, or Cortex-M3/4
0060 - 0114: /7, to a rich feature system running on MIPS32, ARM Cor
Total: [0115]Bytes
Receive, len[0037]:
0000 - 0036: tex-A8, ARM Cortex-A9 DualCore etc.
Total: [0037]Bytes
msh />
```
分多次下载得到的数据,与通用的 GET 请求获取的数据完全一致,分片下载功能正常。
- 在 MSH 中使用命令 `web_get_test -u [URI]` 格式命令执行 GET 请求示例程序,其中 URI 为用户自定义的支持 GET 请求的地址。

@ -229,17 +229,16 @@ while(1)
webclient_close(session);
```
- 发送获取部分数据的 GET 请求(多用于断点续传)
- 发送获取部分数据的 GET 请求(多用于断点续传/分片下载
```c
struct webclient_session *session = NULL;
session = webclient_create(1024);
if(webclient_get_position(URI, 100) != 206)
{
LOG_E("error!");
}
webclient_connect(session, URI);
webclient_header_fields_add(session, "Range: bytes=%d-%d\r\n", 0, 99);
webclient_send_header(session, WEBCLIENT_GET);
while(1)
{

@ -71,6 +71,7 @@ enum WEBCLIENT_METHOD
WEBCLIENT_USER_METHOD,
WEBCLIENT_GET,
WEBCLIENT_POST,
WEBCLIENT_HEAD
};
struct webclient_header
@ -95,6 +96,7 @@ struct webclient_session
int content_length;
size_t content_remainder; /* remainder of content length */
int (*handle_function)(char *buffer, int size); /* handle function */
rt_bool_t is_tls; /* HTTPS connect */
#ifdef WEBCLIENT_USING_MBED_TLS
@ -107,7 +109,9 @@ struct webclient_session *webclient_session_create(size_t header_sz);
/* send HTTP GET request */
int webclient_get(struct webclient_session *session, const char *URI);
int webclient_get_position(struct webclient_session *session, const char *URI, int position);
int webclient_shard_position_function(struct webclient_session *session, const char *URI, int size);
char *webclient_register_shard_position_function(struct webclient_session *session, int (*handle_function)(char *buffer, int size));
/* send HTTP POST request */
int webclient_post(struct webclient_session *session, const char *URI, const void *post_data, size_t data_len);

@ -0,0 +1,137 @@
/*
* Copyright (c) 2006-2018, RT-Thread Development Team
*
* SPDX-License-Identifier: Apache-2.0
*
* Change Logs:
* Date Author Notes
* 2021-06-03 xiangxistu the first version
*/
#include <rtthread.h>
#include <webclient.h>
#include "stdlib.h"
#define GET_LOCAL_URI "http://www.rt-thread.com/service/rt-thread.txt"
#define CHARACTER_LENGTH 60
/* handle function, you can store data and so on */
static int shard_download_handle(char *buffer, int length)
{
int outindex, inindex = 0;
int boundary;
/* print the receive data */
rt_kprintf("\nReceive, len[%04d]:\n", length);
for (outindex = 0; outindex < length; outindex = outindex + inindex)
{
char print_buffer[CHARACTER_LENGTH + 1] = {0};
char *point = RT_NULL;
point = print_buffer;
if(length - outindex > CHARACTER_LENGTH)
{
boundary = CHARACTER_LENGTH;
}
else
{
boundary = length - outindex;
}
for (inindex = 0; inindex < boundary; inindex++)
{
*point++ = buffer[outindex + inindex];
}
*point = 0;
rt_kprintf("%04d - %04d: %s\n", outindex, outindex + boundary - 1, print_buffer);
}
rt_kprintf("Total: [%04d]Bytes\n", length);
/* release this buffer if we have handled data */
web_free(buffer);
return RT_EOK;
}
int webclient_shard_download_test(int argc, char **argv)
{
struct webclient_session* session = RT_NULL;
rt_err_t result = RT_EOK;
char *uri = RT_NULL;
int size = 0;
int usage_flag = 0;
if (argc == 1)
{
uri = web_strdup(GET_LOCAL_URI);
}
else
{
int index;
for(index = 1; index < argc; index = index + 2)
{
if(rt_strstr(argv[index], "-u"))
{
uri = web_strdup(argv[index + 1]);
}
else if(rt_strstr(argv[index], "-l"))
{
size = atoi(argv[index + 1]);
}
else
{
usage_flag = 1;
break;
}
}
}
if(usage_flag)
{
rt_kprintf("web_shard_test -u [URI] - webclient HEAD and GET request test.\n");
rt_kprintf("web_shard_test -l [SIZE] - the length of receive buffer.\n");
return -RT_ERROR;
}
if(uri == RT_NULL)
{
uri = web_strdup(GET_LOCAL_URI);
}
/* sometime, the header bufsz can set more smaller */
session = webclient_session_create(WEBCLIENT_HEADER_BUFSZ / 4);
if (session == RT_NULL)
{
result = -RT_ENOMEM;
goto __exit;
}
/* register the handle function, you can handle data in the function */
webclient_register_shard_position_function(session, shard_download_handle);
/* the "memory size" that you can provide in the project and uri */
webclient_shard_position_function(session, uri, size);
/* clear the handle function */
webclient_register_shard_position_function(session, RT_NULL);
__exit:
if (uri)
{
web_free(uri);
}
if (session)
{
webclient_close(session);
}
return result;
}
#ifdef FINSH_USING_MSH
#include <finsh.h>
MSH_CMD_EXPORT_ALIAS(webclient_shard_download_test, web_shard_test, webclient head and get request test);
#endif /* FINSH_USING_MSH */

@ -12,6 +12,7 @@
* 2018-01-04 aozima add ipv6 address support.
* 2018-07-26 chenyong modify log information
* 2018-08-07 chenyong modify header processing
* 2021-06-09 xiangxistu add shard download function
*/
#include <stdio.h>
@ -626,7 +627,7 @@ static int webclient_send_header(struct webclient_session *session, int method)
header = session->header->buffer;
if (session->header->length == 0)
if (session->header->length == 0 && method <= WEBCLIENT_GET)
{
/* use default header data */
if (webclient_header_fields_add(session, "GET %s HTTP/1.1\r\n", session->req_url) < 0)
@ -663,6 +664,9 @@ static int webclient_send_header(struct webclient_session *session, int method)
else if (method == WEBCLIENT_POST)
length = rt_snprintf(session->header->buffer, session->header->size, "POST %s HTTP/1.1\r\n%s",
session->req_url ? session->req_url : "/", header_buffer);
else if (method == WEBCLIENT_HEAD)
length = rt_snprintf(session->header->buffer, session->header->size, "HEAD %s HTTP/1.1\r\n%s",
session->req_url ? session->req_url : "/", header_buffer);
session->header->length = length;
web_free(header_buffer);
@ -963,19 +967,36 @@ int webclient_get(struct webclient_session *session, const char *URI)
}
/**
* http breakpoint resume.
* register a handle function for http breakpoint resume and shard download.
*
* @param function
*
* @return the pointer
*/
char *webclient_register_shard_position_function(struct webclient_session *session, int (*handle_function)(char *buffer, int size))
{
session->handle_function = handle_function;
return (char *)session->handle_function;
}
/**
* http breakpoint resume and shard download.
*
* @param session webclient session
* @param URI input server URI address
* @param position last downloaded position
* @param the buffer size that you alloc
*
* @return <0: send GET request failed
* >0: response http status code
*/
int webclient_get_position(struct webclient_session *session, const char *URI, int position)
int webclient_shard_position_function(struct webclient_session *session, const char *URI, int size)
{
int rc = WEBCLIENT_OK;
int resp_status = 0;
int real_total_len = 0;
int start_position, end_position = 0;
char *buffer = RT_NULL;
RT_ASSERT(session);
RT_ASSERT(URI);
@ -986,13 +1007,38 @@ int webclient_get_position(struct webclient_session *session, const char *URI, i
return rc;
}
/* splice header*/
if (webclient_header_fields_add(session, "Range: bytes=%d-\r\n", position) <= 0)
rc = webclient_send_header(session, WEBCLIENT_HEAD);
if (rc != WEBCLIENT_OK)
{
rc = -WEBCLIENT_ERROR;
return rc;
}
/* handle the response header of webclient server by HEAD request */
resp_status = webclient_handle_response(session);
if(resp_status >= 0)
{
real_total_len = webclient_content_length_get(session);
LOG_D("The length[%04d] of real data of URI.", real_total_len);
}
/* clean header buffer and size */
rt_memset(session->header->buffer, 0x00, session->header->size);
session->header->length = 0;
for(start_position = end_position; end_position < real_total_len; start_position = end_position + 1)
{
RT_ASSERT(start_position < real_total_len);
int data_len = 0;
end_position = start_position + size - 1;
if(end_position >= real_total_len)
{
end_position = real_total_len;
}
/* splice header and send header */
LOG_I("Range: [%04d -> %04d]", start_position, end_position);
webclient_header_fields_add(session, "Range: bytes=%d-%d\r\n", start_position, end_position);
rc = webclient_send_header(session, WEBCLIENT_GET);
if (rc != WEBCLIENT_OK)
{
@ -1003,7 +1049,6 @@ int webclient_get_position(struct webclient_session *session, const char *URI, i
resp_status = webclient_handle_response(session);
LOG_D("get position handle response(%d).", resp_status);
if (resp_status > 0)
{
const char *location = webclient_header_fields_get(session, "Location");
@ -1025,13 +1070,22 @@ int webclient_get_position(struct webclient_session *session, const char *URI, i
session->header->length = 0;
rt_memset(session->header->buffer, 0, session->header->size);
rc = webclient_get_position(session, new_url, position);
rc = webclient_shard_position_function(session, new_url, size);
web_free(new_url);
return rc;
}
}
/* receive the incoming data */
data_len = webclient_response(session, &buffer, RT_NULL);
session->handle_function(buffer, data_len);
/* clean header buffer and size */
rt_memset(session->header->buffer, 0x00, session->header->size);
session->header->length = 0;
}
return resp_status;
}

Loading…
Cancel
Save