source: trunk/zoo-project/zoo-kernel/ulinet.c @ 797

Last change on this file since 797 was 797, checked in by djay, 4 years ago

Return an ExceptionReport? in case any inpts failed to be downloaded.

  • Property svn:eol-style set to native
  • Property svn:mime-type set to text/x-csrc
File size: 14.9 KB
Line 
1/*
2 *  ulinet.c
3 *
4 * Author : Gérald FENOY
5 *
6 * Copyright (c) 2008-2015 GeoLabs SARL
7 *
8 * Permission is hereby granted, free of charge, to any person obtaining a copy
9 * of this software and associated documentation files (the "Software"), to deal
10 * in the Software without restriction, including without limitation the rights
11 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
12 * copies of the Software, and to permit persons to whom the Software is
13 * furnished to do so, subject to the following conditions:
14 *
15 * The above copyright notice and this permission notice shall be included in
16 * all copies or substantial portions of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
21 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
22 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
23 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
24 * THE SOFTWARE.
25 *
26 */
27
28#define _ULINET
29#define MAX_WAIT_MSECS 180*1000 /* Wait max. 180 seconds */
30#include "ulinet.h"
31#include <assert.h>
32
33/**
34 * Write the downloaded content to a _HINTERNET structure
35 *
36 * @param buffer the buffer to read
37 * @param size size of each member
38 * @param nmemb number of element to read
39 * @param data the _HINTERNET structure to write in
40 * @return the size red, -1 if buffer is NULL
41 */
42size_t write_data_into(void *buffer, size_t size, size_t nmemb, void *data){
43  size_t realsize = size * nmemb;
44  _HINTERNET *psInternet;
45  if(buffer==NULL){
46    buffer=NULL;
47    return -1;
48  }
49  psInternet=(_HINTERNET *)data;
50  if(psInternet->pabyData){
51    psInternet->pabyData=(unsigned char*)realloc(psInternet->pabyData,psInternet->nDataLen+realsize+1);
52    psInternet->nDataAlloc+=psInternet->nDataLen+realsize+1;
53  }
54  else{
55    psInternet->pabyData=(unsigned char*)malloc(psInternet->nDataLen+realsize+1);
56    psInternet->nDataAlloc=realsize+1;
57  }
58
59  if (psInternet->pabyData) {
60    memcpy( psInternet->pabyData + psInternet->nDataLen, buffer, realsize);
61    psInternet->nDataLen += realsize;
62    psInternet->pabyData[psInternet->nDataLen] = 0;
63  }
64
65  buffer=NULL;
66  return realsize;
67}
68
69/**
70 * In case of presence of "Set-Cookie" in the headers red, store the cookie
71 * identifier in CCookie
72 *
73 * @param buffer the buffer to read
74 * @param size size of each member
75 * @param nmemb number of element to read
76 * @param data the _HINTERNET structure to write in
77 * @return the size red, -1 if buffer is NULL
78 * @see CCookie
79 */
80size_t header_write_data(void *buffer, size_t size, size_t nmemb, void *data){
81  if(strncmp("Set-Cookie: ",(char*)buffer,12)==0){
82    int i;
83    char env[256];
84    char path[256];
85    char domain[256];
86    char* tmp;
87    for(i=0;i<12;i++)
88#ifndef WIN32
89      buffer++;
90#else
91        ;
92#endif
93    sscanf((char*)buffer,"%s; path=%s; domain=%s",env,path,domain);
94    _HINTERNET *psInternet=(_HINTERNET *)data;
95    tmp=strcat(env,CCookie[psInternet->id][0]);
96#ifdef MSG_LAF_OUT
97    printf("\n**Cookie env : [%s] , path : [%s], domain : [%s]**\n",env,path,domain);
98    printf("buffer : %d (%s) (%s) (%s)\n",(buffer==NULL),buffer,tmp,CCookie);
99#endif
100    strcpy(CCookie[psInternet->id][0],tmp);
101  }
102  return size * nmemb;//write_data_into(buffer,size,nmemb,data,HEADER);
103};
104
105/**
106 * Define the proxy to use for a CURL handler
107 *
108 * @param handle the CURL handler
109 * @param host the proxy host (including http://)
110 * @param port the proxy port
111 */
112void setProxy(CURL* handle,char* host,long port){
113  char* proxyDef=(char*)malloc((strlen(host)+10+2)*sizeof(char));
114  sprintf(proxyDef,"%s:%ld",host,port);
115  curl_easy_setopt(handle,CURLOPT_PROXY,proxyDef);
116  free(proxyDef);
117}
118
119/**
120 * MACOSX
121 */
122#if defined(macintosh) || (defined(__MACH__) && defined(__APPLE__))
123
124
125char* CFStringToCString(CFStringRef dest,char *buffer){
126  CFStringEncoding encoding = kCFStringEncodingUTF8;
127  Boolean bool2 = CFStringGetCString(dest,buffer,1024,encoding);
128  if(bool2){
129    printf("Loaded into local_buffer");
130    return buffer;
131  }
132  return NULL;
133}
134
135OSStatus setProxiesForProtcol(CURL* handle,const char *proto){
136  OSStatus              err;
137  CFDictionaryRef proxyDict;
138  CFArrayRef            proxies;
139 
140  CFStringRef key_enabled = NULL;
141  CFStringRef key_host = NULL;
142  CFStringRef key_port = NULL;
143 
144  bool proxy_enabled;
145  char *proxy_host;
146  long proxy_port;
147 
148  proxyDict = NULL;
149  proxies = NULL;
150
151  err = noErr;
152  proxyDict = SCDynamicStoreCopyProxies(NULL);
153
154  if(strncmp(proto,"http",4)==0){
155      key_enabled=kSCPropNetProxiesHTTPEnable;
156      key_host=kSCPropNetProxiesHTTPProxy;
157      key_port=kSCPropNetProxiesHTTPPort;
158  }
159  else
160    if(strncmp(proto,"https",5)==0){
161      key_enabled=kSCPropNetProxiesHTTPSEnable;
162      key_host=kSCPropNetProxiesHTTPSProxy;
163      key_port=kSCPropNetProxiesHTTPSPort;
164    }
165
166  CFNumberGetValue(CFDictionaryGetValue(proxyDict,key_enabled),kCFNumberIntType,&proxy_enabled);
167  if(proxy_enabled){
168    CFNumberGetValue(CFDictionaryGetValue(proxyDict,key_port),CFNumberGetType(CFDictionaryGetValue(proxyDict,key_port)),&proxy_port);
169    char buffer[1024];
170    CFStringToCString(CFDictionaryGetValue(proxyDict,key_host),buffer);
171    proxy_host=buffer;
172
173#ifdef MSG_LAF_VERBOSE
174    printf("\n**[PROXY SETTINGS DETECTION %s (%d) %s:%li (%s)]**\n",proto,proxy_enabled,(char*)proxy_host,proxy_port,buffer);
175#endif
176
177    if (proxyDict == NULL) {
178      err = coreFoundationUnknownErr;
179    }
180
181    setProxy(handle,proxy_host,proxy_port);
182  }
183  return err;
184}
185#else
186/**
187 * Should autodetect the proxy configuration (do nothing on linux)
188 *
189 * @param handle a CURL handle
190 * @param proto the protocol requiring the use of a proxy
191 */
192bool setProxiesForProtcol(CURL* handle,const char *proto){
193#ifdef MSG_LAF_VERBOSE
194  fprintf( stderr, "setProxiesForProtocol (do nothing) ...\n" );
195#endif
196  return true;
197}
198#endif
199
200/**
201 * Create a HINTERNET
202 *
203 * @param lpszAgent the HTPP User-Agent to use to send requests
204 * @param  dwAccessType type of access required
205 * @param  lpszProxyName the name of the proxy server(s) to use
206 * @param  lpszProxyBypass ip address or host names which should not be routed
207 *  through the proxy
208 * @param  dwFlags Options (INTERNET_FLAG_ASYNC,INTERNET_FLAG_FROM_CACHE,INTERNET_FLAG_OFFLINE)
209 * @return the created HINTERNET
210 */
211HINTERNET InternetOpen(char* lpszAgent,int dwAccessType,char* lpszProxyName,char* lpszProxyBypass,int dwFlags){
212  HINTERNET ret;
213  ret.handle=curl_multi_init();
214  ret.agent=strdup(lpszAgent);
215  ret.nb=0;
216  ret.ihandle[ret.nb].header=NULL;
217  return ret;
218}
219
220/**
221 * Close a HINTERNET connection and free allocated resources
222 *
223 * @param handle0 the HINTERNET connection to close
224 */
225void InternetCloseHandle(HINTERNET* handle0){
226  int i=0;
227  for(i=0;i<handle0->nb;i++){
228    _HINTERNET handle=handle0->ihandle[i];
229    if(handle.hasCacheFile>0){
230      fclose(handle.file);
231      unlink(handle.filename);
232    }
233    else{
234      handle.pabyData = NULL;
235      handle.nDataAlloc = handle.nDataLen = 0;
236    }
237    if(handle0->ihandle[i].header!=NULL){
238      curl_slist_free_all(handle0->ihandle[i].header);
239      handle0->ihandle[i].header=NULL;
240    }
241    if(handle.post!=NULL)
242      free(handle.post);
243    free(handle.mimeType);
244    handle.mimeType = NULL;
245  }
246  if(handle0->handle)
247    curl_multi_cleanup(handle0->handle);
248  free(handle0->agent);
249  for(i=handle0->nb-1;i>=0;i--){
250    free(handle0->waitingRequests[i]);
251  }
252}
253
254/**
255 * Create a new element in the download queue
256 *
257 * @param hInternet the HINTERNET connection to add the download link
258 * @param lpszUrl the url to download
259 * @param lpszHeaders the additional headers to be sent to the HTTP server
260 * @param dwHeadersLength the size of the additional headers
261 * @param dwFlags desired download mode (INTERNET_FLAG_NO_CACHE_WRITE for not using cache file)
262 * @param dwContext not used
263 */
264HINTERNET InternetOpenUrl(HINTERNET* hInternet,LPCTSTR lpszUrl,LPCTSTR lpszHeaders,size_t dwHeadersLength,size_t dwFlags,size_t dwContext){
265
266  char filename[255];
267  struct MemoryStruct header;
268
269  hInternet->ihandle[hInternet->nb].handle=curl_easy_init( );
270  hInternet->ihandle[hInternet->nb].hasCacheFile=0;
271  hInternet->ihandle[hInternet->nb].nDataAlloc = 0;
272  hInternet->ihandle[hInternet->nb].mimeType = NULL;
273  hInternet->ihandle[hInternet->nb].nDataLen = 0;
274  hInternet->ihandle[hInternet->nb].id = hInternet->nb;
275  hInternet->ihandle[hInternet->nb].nDataAlloc = 0;
276  hInternet->ihandle[hInternet->nb].pabyData = NULL;
277  hInternet->ihandle[hInternet->nb].post = NULL;
278
279  curl_easy_setopt(hInternet->ihandle[hInternet->nb].handle, CURLOPT_COOKIEFILE, "ALL");
280#ifndef TIGER
281  curl_easy_setopt(hInternet->ihandle[hInternet->nb].handle, CURLOPT_COOKIELIST, "ALL");
282#endif
283  curl_easy_setopt(hInternet->ihandle[hInternet->nb].handle, CURLOPT_USERAGENT, hInternet->agent);
284 
285  curl_easy_setopt(hInternet->ihandle[hInternet->nb].handle,CURLOPT_FOLLOWLOCATION,1);
286  curl_easy_setopt(hInternet->ihandle[hInternet->nb].handle,CURLOPT_MAXREDIRS,3);
287 
288  header.memory=NULL;
289  header.size = 0;
290
291  curl_easy_setopt(hInternet->ihandle[hInternet->nb].handle, CURLOPT_HEADERFUNCTION, header_write_data);
292  curl_easy_setopt(hInternet->ihandle[hInternet->nb].handle, CURLOPT_WRITEHEADER, (void *)&header);
293
294#ifdef MSG_LAF_VERBOSE
295  curl_easy_setopt(hInternet->ihandle[hInternet->nb].handle, CURLOPT_VERBOSE, 1);
296#endif
297
298     
299  switch(dwFlags)
300    {
301    case INTERNET_FLAG_NO_CACHE_WRITE:
302      hInternet->ihandle[hInternet->nb].hasCacheFile=-1;
303      curl_easy_setopt(hInternet->ihandle[hInternet->nb].handle, CURLOPT_WRITEFUNCTION, write_data_into);
304      curl_easy_setopt(hInternet->ihandle[hInternet->nb].handle, CURLOPT_WRITEDATA, (void*)&hInternet->ihandle[hInternet->nb]);
305      break;
306    default:
307      sprintf(hInternet->ihandle[hInternet->nb].filename,"/tmp/ZOO_Cache%d",(int)time(NULL));
308      hInternet->ihandle[hInternet->nb].filename[24]=0;
309#ifdef MSG_LAF_VERBOSE
310      fprintf(stderr,"file=%s",hInternet->ihandle[hInternet->nb].filename);
311#endif
312      hInternet->ihandle[hInternet->nb].filename=filename;
313      hInternet->ihandle[hInternet->nb].file=fopen(hInternet->ihandle[hInternet->nb].filename,"w+");
314   
315      hInternet->ihandle[hInternet->nb].hasCacheFile=1;
316      curl_easy_setopt(hInternet->ihandle[hInternet->nb].handle, CURLOPT_WRITEFUNCTION, NULL);
317      curl_easy_setopt(hInternet->ihandle[hInternet->nb].handle, CURLOPT_WRITEDATA, hInternet->ihandle[hInternet->nb].file);
318      hInternet->ihandle[hInternet->nb].nDataLen=0;
319      break;
320    }
321#ifdef ULINET_DEBUG
322  fprintf(stderr,"URL (%s)\nBODY (%s)\n",lpszUrl,lpszHeaders);
323#endif
324  if(lpszHeaders!=NULL && strlen(lpszHeaders)>0){
325#ifdef MSG_LAF_VERBOSE
326    fprintf(stderr,"FROM ULINET !!");
327    fprintf(stderr,"HEADER : [%s] %d\n",lpszHeaders,dwHeadersLength);
328#endif
329    curl_easy_setopt(hInternet->ihandle[hInternet->nb].handle,CURLOPT_POST,1);
330#ifdef ULINET_DEBUG
331    fprintf(stderr,"** (%s) %d **\n",lpszHeaders,dwHeadersLength);
332    curl_easy_setopt(hInternet->ihandle[hInternet->nb].handle,CURLOPT_VERBOSE,1);
333#endif
334    hInternet->ihandle[hInternet->nb].post=strdup(lpszHeaders);
335    curl_easy_setopt(hInternet->ihandle[hInternet->nb].handle,CURLOPT_POSTFIELDS,hInternet->ihandle[hInternet->nb].post);
336    curl_easy_setopt(hInternet->ihandle[hInternet->nb].handle,CURLOPT_POSTFIELDSIZE,(long)dwHeadersLength);
337  }
338  if(hInternet->ihandle[hInternet->nb].header!=NULL)
339    curl_easy_setopt(hInternet->ihandle[hInternet->nb].handle,CURLOPT_HTTPHEADER,hInternet->ihandle[hInternet->nb].header);
340
341  curl_easy_setopt(hInternet->ihandle[hInternet->nb].handle,CURLOPT_URL,lpszUrl);
342
343  curl_multi_add_handle(hInternet->handle,hInternet->ihandle[hInternet->nb].handle);
344 
345  hInternet->ihandle[hInternet->nb].header=NULL;
346  ++hInternet->nb;
347
348#ifdef ULINET_DEBUG
349  fprintf(stderr,"DEBUG MIMETYPE: %s\n",hInternet.mimeType);
350  fflush(stderr);
351#endif
352  return *hInternet;
353};
354
355/**
356 * Download all opened urls in the queue
357 *
358 * @param hInternet the HINTERNET structure containing the queue
359 * @return 0
360 */
361int processDownloads(HINTERNET* hInternet){
362  int still_running=0;
363  int msgs_left=0;
364  int i=0;
365  do{
366    curl_multi_perform(hInternet->handle, &still_running);
367  }while(still_running); 
368  for(i=0;i<hInternet->nb;i++){
369    char *tmp;
370    curl_easy_getinfo(hInternet->ihandle[i].handle,CURLINFO_CONTENT_TYPE,&tmp);
371    if(tmp!=NULL)
372      hInternet->ihandle[i].mimeType=strdup(tmp);
373    curl_easy_getinfo(hInternet->ihandle[i].handle,CURLINFO_RESPONSE_CODE,&hInternet->ihandle[i].code);
374    curl_multi_remove_handle(hInternet->handle, hInternet->ihandle[i].handle);
375    curl_easy_cleanup(hInternet->ihandle[i].handle);
376  }
377  return 0;
378}
379
380/**
381 * Initialize the CCookie for a specific index (hInternet.nb)
382 *
383 * @param hInternet the HINTERNET structure to know the CCookie index to reset
384 * @return 1
385 * @see HINTERNET
386 */
387int freeCookieList(HINTERNET hInternet){
388  memset(&CCookie[hInternet.nb][0],0,1024);
389#ifndef TIGER
390  curl_easy_setopt(hInternet.ihandle[hInternet.nb].handle, CURLOPT_COOKIELIST, "ALL");
391#endif
392  return 1;
393}
394
395/**
396 * Copy a downloaded content
397 *
398 * @param hInternet the _HINTERNET structure
399 * @param lpBuffer the memory space to copy the downloaded content
400 * @param dwNumberOfBytesToRead the size of lpBuffer
401 * @param lpdwNumberOfBytesRead number of bytes red
402 * @return 1 on success, 0 if failure
403 */
404int InternetReadFile(_HINTERNET hInternet,LPVOID lpBuffer,int dwNumberOfBytesToRead, size_t *lpdwNumberOfBytesRead){
405  int dwDataSize;
406
407  if(hInternet.hasCacheFile>0){
408    fseek (hInternet.file , 0 , SEEK_END);
409    dwDataSize=ftell(hInternet.file); //taille du ficher
410    rewind (hInternet.file);
411  }
412  else{
413    memset(lpBuffer,0,hInternet.nDataLen+1);
414    memcpy(lpBuffer, hInternet.pabyData, hInternet.nDataLen );
415    dwDataSize=hInternet.nDataLen;
416    free( hInternet.pabyData );
417    hInternet.pabyData=NULL;
418  }
419
420  if( dwNumberOfBytesToRead /* buffer size */ < dwDataSize )
421    return 0;
422
423#ifdef MSG_LAF_VERBOSE
424  printf("\nfile size : %dko\n",dwDataSize/1024);
425#endif
426
427  if(hInternet.hasCacheFile>0){
428    *lpdwNumberOfBytesRead = fread(lpBuffer,1,dwDataSize,hInternet.file); 
429  }
430  else{
431    *lpdwNumberOfBytesRead = hInternet.nDataLen;
432    free( hInternet.pabyData );
433    hInternet.pabyData = NULL;
434    hInternet.nDataAlloc = hInternet.nDataLen = 0;
435  }
436
437  CCookie[hInternet.id][0]=0;
438
439  if( *lpdwNumberOfBytesRead < dwDataSize )
440      return 0;
441  else
442      return 1; // TRUE
443}
444
445
446/**
447 * Use basic authentication for accessing a resource
448 *
449 * @param hInternet the _HINTERNET structure
450 * @param login the login to use to authenticate
451 * @param passwd the password to use to authenticate
452 */
453int setBasicAuth(HINTERNET hInternet,char* login,char* passwd){
454  char *tmp;
455  tmp=(char*)malloc((strlen(login)+strlen(passwd)+2)*sizeof(char));
456  sprintf(tmp,"%s:%s",login,passwd);
457  if(curl_easy_setopt(hInternet.ihandle[hInternet.nb].handle,CURLOPT_USERPWD,tmp)==CURLE_OUT_OF_MEMORY){
458    free(tmp);
459    return -1;
460  }
461  curl_easy_setopt(hInternet.ihandle[hInternet.nb].handle, CURLOPT_HTTPAUTH,CURLAUTH_ANY);
462  free(tmp);
463  return 0;
464}
Note: See TracBrowser for help on using the repository browser.

Search

ZOO Sponsors

http://www.zoo-project.org/trac/chrome/site/img/geolabs-logo.pnghttp://www.zoo-project.org/trac/chrome/site/img/neogeo-logo.png http://www.zoo-project.org/trac/chrome/site/img/apptech-logo.png http://www.zoo-project.org/trac/chrome/site/img/3liz-logo.png http://www.zoo-project.org/trac/chrome/site/img/gateway-logo.png

Become a sponsor !

Knowledge partners

http://www.zoo-project.org/trac/chrome/site/img/ocu-logo.png http://www.zoo-project.org/trac/chrome/site/img/gucas-logo.png http://www.zoo-project.org/trac/chrome/site/img/polimi-logo.png http://www.zoo-project.org/trac/chrome/site/img/fem-logo.png http://www.zoo-project.org/trac/chrome/site/img/supsi-logo.png http://www.zoo-project.org/trac/chrome/site/img/cumtb-logo.png

Become a knowledge partner

Related links

http://zoo-project.org/img/ogclogo.png http://zoo-project.org/img/osgeologo.png