这个例子来自参考文献[1], 那里有很多小bug,我都做了修改,在这里不一一说明了。ncurse界面编程比较容易入门,就是几个接口,网上资料很多,这里不详述了。
//gcc -g mget.c -o mget -lcurl -lncurses -lm
//
#include
#include
#include
#include
#include
#include
#include WINDOW *win;
int width, height;WINDOW *create_newwin (int height, int width, int starty, int startx);
void destroy_win (WINDOW * local_win);typedef struct progress
{char byteInterval[64]; //下载分段的字节范围int handle; //文件编号或索引double totalDownloaded; //累计下载double segmentSize;//片段总长字节double currentTime;//当前时间戳double startTime;//开始时间戳
} *dl_progress;size_t write_data (void *ptr, size_t size, size_t nmemb, FILE * stream)
{size_t written;written = fwrite (ptr, size, nmemb, stream);return written;
}static size_t get_size_struct (void *ptr, size_t size, size_t nmemb, void *data)
{(void) ptr;(void) data;// return only the size, dump the restreturn (size_t) (size * nmemb);
}int display_progress (dl_progress pgr)
{int i = pgr->handle;int totalDots = 80; //进度条的长度是80, 剩余的长度是160-80=80double totalDownloaded = pgr->totalDownloaded; //已下载片段大小time_t diffTime = pgr->currentTime - pgr->startTime;double segmentSize = pgr->segmentSize; //总片段大小double averageSpeed = pgr->totalDownloaded / diffTime; //平均下载速度double fractionDownloaded = totalDownloaded / segmentSize; //下载百分比int dots = round (fractionDownloaded * totalDots); //用几个点来表示?// create the metermvwprintw (win, i + 3, 2, "%3.0f%% [%2d] [", fractionDownloaded * 100, i);int ii = 0;for (; ii < dots; ii++)mvwprintw (win, i + 3, 13 + ii, "="); //8for (; ii < totalDots; ii++)mvwprintw (win, i + 3, 13 + ii, " ");mvwprintw (win, i + 3, totalDots + 13, "]");//注意间隔是40// display some download infomvwprintw (win, i + 3, totalDots + 15, "[%s]", pgr->byteInterval);mvwprintw (win, i + 3, totalDots + 45, "%03.2f KB/s", averageSpeed / 1024);mvwprintw (win, i + 3, width - 18, "%0.2f / %0.2f MB", totalDownloaded / 1024 / 1024, segmentSize / 1024 / 1024);wrefresh (win);return 0;
}int progress_func (dl_progress ptr, double totalToDownload, double nowDownloaded, double totalToUpload, double nowUploaded)
{time_t seconds;seconds = time (NULL);ptr->totalDownloaded = nowDownloaded;ptr->currentTime = seconds;return display_progress (ptr);
}static double get_download_size (char *url)
{CURL *curl;CURLcode res;double size = 0.0;curl = curl_easy_init ();curl_easy_setopt (curl, CURLOPT_URL, url);curl_easy_setopt (curl, CURLOPT_NOBODY, 1L);curl_easy_setopt (curl, CURLOPT_HEADERFUNCTION, get_size_struct);curl_easy_setopt (curl, CURLOPT_FOLLOWLOCATION, 1L);res = curl_easy_perform (curl);res = curl_easy_getinfo (curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &size);if (res != CURLE_OK){fprintf (stderr, "curl_easy_getinfo() failed: %s
", curl_easy_strerror (res));}curl_easy_cleanup (curl);return size;
}int main (int argc, char *argv[])
{if (argc != 3){printf ("Incorrect parameters
Usage: %s
", argv[0]);return -1;}// setup the start time for average download speed later when we have finishedtime_t startTime;startTime = time (NULL);// number of partsint parts = strtol (argv[1], &argv[1], 10); // base 10struct progress *MyProgress = malloc(sizeof(struct progress) * parts);memset(MyProgress, 0 , sizeof(struct progress) * parts);// create the windowinitscr ();height = parts + 5;width = 160; //这里窗口的宽度是160int starty = (LINES - height) / 2;int startx = (COLS - width) / 2;refresh ();win = create_newwin (height, width, starty, startx);// setup our varsconst char *outputfile;char *url = argv[2];double partSize = 0;double segLocation = 0;int still_running;int i;// get file nameoutputfile = strrchr ((const char *) url, '/') + 1;// get file sizedouble size = get_download_size (argv[2]);partSize = size / parts;mvwprintw (win, 0, 10, "Downloading %dx%0.2f MB segments (%0.2f MB) %s", parts, size / parts / 1024 / 1024, size / 1024 / 1024, outputfile);// setup curl varsFILE *fileparts[parts];CURL *single_handles[parts];CURLM *multi_handle;CURLMsg *msg;int msgs_left;int error;curl_global_init (CURL_GLOBAL_ALL);for (i = 0; i < parts; i++){time_t seconds;seconds = time (NULL);memset(MyProgress[i].byteInterval, 0, sizeof(MyProgress[i].byteInterval));MyProgress[i].startTime = seconds;MyProgress[i].handle = i;MyProgress[i].segmentSize = partSize;MyProgress[i].totalDownloaded = 0;// setup our output filenamechar filename[50] = { 0 };snprintf (filename, sizeof (filename), "%s.part.%0d", outputfile, i);// allocate curl handle for each segmentsingle_handles[i] = curl_easy_init ();fileparts[i] = fopen (filename, "w");double nextPart = 0;if (i == parts - 1){nextPart = size;}else{nextPart = segLocation + partSize - 1;}char range[64] = { 0 };snprintf (range, sizeof (range), "%12.0f-%12.0f", segLocation, nextPart);memcpy(MyProgress[i].byteInterval, range, strlen(range));// set some curl options.curl_easy_setopt (single_handles[i], CURLOPT_URL, url);curl_easy_setopt (single_handles[i], CURLOPT_RANGE, range); //设置range请求curl_easy_setopt (single_handles[i], CURLOPT_FOLLOWLOCATION, 1L);curl_easy_setopt (single_handles[i], CURLOPT_WRITEFUNCTION, write_data);//设置接收到文件内容回调curl_easy_setopt (single_handles[i], CURLOPT_WRITEDATA, fileparts[i]); //写数据回调的最后一个参数curl_easy_setopt (single_handles[i], CURLOPT_NOPROGRESS, 0); //设置进度回调功能curl_easy_setopt (single_handles[i], CURLOPT_PROGRESSFUNCTION, progress_func); //设置进度回调函数curl_easy_setopt (single_handles[i], CURLOPT_PROGRESSDATA, &MyProgress[i]); //设置进度回调函数的第一个参数segLocation += partSize;}multi_handle = curl_multi_init ();// add all individual transfers to the stackfor (i = 0; i < parts; i++){curl_multi_add_handle (multi_handle, single_handles[i]);}curl_multi_perform (multi_handle, &still_running);do{struct timeval timeout;int rc; // return codefd_set fdread;fd_set fdwrite;fd_set fdexcep; // file descriptor exceptionint maxfd = -1;long curl_timeo = -1;FD_ZERO (&fdread);FD_ZERO (&fdwrite);FD_ZERO (&fdexcep);// set a suitable timeout to play withtimeout.tv_sec = 100 * 1000;timeout.tv_usec = 0;curl_multi_timeout (multi_handle, &curl_timeo);if (curl_timeo >= 0){timeout.tv_sec = curl_timeo / 1000;if (timeout.tv_sec > 1){timeout.tv_sec = 1;}else{timeout.tv_usec = (curl_timeo % 1000) * 1000;}}curl_multi_fdset (multi_handle, &fdread, &fdwrite, &fdexcep, &maxfd);rc = select (maxfd + 1, &fdread, &fdwrite, &fdexcep, &timeout);switch (rc){case -1:fprintf (stderr, "Could not select the error
");break;case 0:/* timeout */default:curl_multi_perform (multi_handle, &still_running);break;}}while (still_running);while ((msg = curl_multi_info_read (multi_handle, &msgs_left))){if (msg->msg == CURLMSG_DONE){int index, found = 0;for (index = 0; index < parts; index++){found = (msg->easy_handle == single_handles[index]);if (found)break;}}}// clean up our multi handlecurl_multi_cleanup (multi_handle);// free up the curl handlesfor (i = 0; i < parts; i++){curl_easy_cleanup (single_handles[i]);}// close ncurses windowdestroy_win(win);endwin ();free(MyProgress);// send some output to the console for records sake.time_t endTime;endTime = time (NULL);printf ("Downloaded %0.2f MB in %ld seconds
", partSize * parts / 1024 / 1024, endTime - startTime);printf ("%0.2f KB/s (average)
", (partSize * parts / (endTime - startTime) / 1024));return 0;
}WINDOW *create_newwin (int height, int width, int starty, int startx)
{WINDOW *local_win;local_win = newwin (height, width, starty, startx);box (local_win, 0, 0);wrefresh (local_win);return local_win;
}void destroy_win (WINDOW * local_win)
{/* box(local_win, ' ', ' '); : This won't produce the desired* result of erasing the window. It will leave it's four corners* and so an ugly remnant of window.*/wborder (local_win, '|', '|', '-', '-', '+', '+', '+', '+');/* The parameters taken are* 1. win: the window on which to operate* 2. ls: character to be used for the left side of the window* 3. rs: character to be used for the right side of the window* 4. ts: character to be used for the top side of the window* 5. bs: character to be used for the bottom side of the window* 6. tl: character to be used for the top left corner of the window* 7. tr: character to be used for the top right corner of the window* 8. bl: character to be used for the bottom left corner of the window* 9. br: character to be used for the bottom right corner of the window*/wrefresh (local_win);delwin (local_win);
}
运行方法
需要两个参数,第一个参数是,分段个数,第二个参数是大文件的链接。默认是大文件下载,上G的文件,可以分为几十兆一个分段,并行异步下载(注意是单线程异步批量,不是多线程),效果还可以。
./mget 20 "http://cdimage.ubuntu.com/releases/14.04/release/ubuntu-14.04-desktop-amd64+mac.iso"
Downloaded 962.00 MB in 867 seconds
1136.20 KB/s (average)
运行截图
附加说明
设计UI界面是个费时费力的细致活, 需要根据屏幕分辨率设计好. 我的ThinkPad是720P的屏,当前的效果给出的是80宽度的框, 进度条使用"="表示, 宽度是40, 高度是文件个数,加上5行空白, 上面是3, 下面是2.一般的笔记本通常没有高的分辨率,建议框的宽度是40就可以,其他的调整按照参考文献[1]设置就够用了。
待解决的问题
如何将这几个文件的即时速度的和统计出来?我暂时没有找到较好的方法.希望大牛指点.
参考文献
[1].https://github.com/logikaljay/mget
在线上服务器上执行下面的命令 curl -vo /dev/null 'http://120.52.72.46:80/fileshare3010.dfiles.eu/c3pr90ntcsf0/auth-1375626538db3c073c81647e872cab8f-210.186.189.166-676861082-14640452...
使用libcurl多线程下载大文件的基本思想: 首选打开文件,将文件等分为指定的片段,使用http range下载,一个线程下载一个片段,当线程下载片段时,它们将数据写到打开文件的指定位置,类似BT文件下载的方式(这样片段下载完成后不用再合并),当所有的子线程下载完成后,这个大文件也就随之下载完成了。 下面是相关源码: //g...
在下面的源码中,我将展示如何使用libcurl提供的进度条功能,以及如何发送range请求, 同时提供了限速功能。
源码如下:
//g++ -g curl_range.cpp -o curl_range -lcurl -lm
//
#include
这是一个简单的获取远程文件大小的源码,我们可以改写为大批量异步的形式.
#include
上篇笔记中梳理了一把 resolver 和 balancer,这里顺着前面的流程走一遍入口的 ClientConn 对象。ClientConn// ClientConn represents a virtual connection to a conceptual endpoint, to // perform RPCs. // //...
我的实验是基于PSPNet模型实现二维图像的语义分割,下面的代码直接从得到的h5文件开始往下做。。。 也不知道是自己的检索能力出现了问题还是咋回事,搜遍全网都没有可以直接拿来用的语义分割代码,东拼西凑,算是搞成功了。 实验平台:Windows、VS2015、Tensorflow1.8 api、Python3.6 具体的流程为:...
Path Tracing 懒得翻译了,相信搞图形学的人都能看得懂,2333 Path Tracing is a rendering algorithm similar to ray tracing in which rays are cast from a virtual camera and traced through a s...
configure_file(
直接复制以下代码创建一个名为settings.xml的文件,放到C:UsersAdministrator.m2下即可