/////////////////////////////////////////////////////////////////////////////////////// // BiffSocko // getTicker.cpp // // This program connects to the www.eoddata.com Website and downloads all the // HTML pages that contain the Ticker Symbols within several different exchanges. // // // This program is free software; you can redistribute it and/or // modify it in any way you want as long as you give the author credit for his part // of the product. // // This program is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. /////////////////////////////////////////////////////////////////////////////////////// #include #include #include #include #include #include #include #include #include #include #define PORT 80 #define MAXDATASIZE 1024 #define TickerFile "ticker" #define NUM 1024 #define SERVER "www.eoddata.com" ///////////////////////////////// // sockclient class ///////////////////////////////// class sockclient{ public: int opensock(char*); }; ///////////////////////////////// // create a socket and connect to // the server ///////////////////////////////// int sockclient::opensock(char *host){ int sockfd; struct hostent *he; struct sockaddr_in their_addr; // connector's address information if ((he=gethostbyname(host)) == NULL) { // get the host info perror("gethostbyname"); exit(1); } if ((sockfd = socket(AF_INET, SOCK_STREAM, 0)) == -1) { perror("socket"); exit(1); } their_addr.sin_family = AF_INET; // host byte order their_addr.sin_port = htons(PORT); // short, network byte order their_addr.sin_addr = *((struct in_addr *)he->h_addr); memset(&(their_addr.sin_zero), '\0', 8); // zero the rest of the struct if (connect(sockfd, (struct sockaddr *)&their_addr, sizeof(struct sockaddr)) == -1) { perror("connect"); exit(1); } return(sockfd); } ///////////////////////////// // getTickers() ///////////////////////////// int getTickers(char *url, FILE *outfile){ char buf[MAXDATASIZE]; sockclient sock1; int descriptor; FILE *sockin, *sockout; char *header; ///////////////////////////////////////// // This is the HTTP Header ///////////////////////////////////////// header="Host: www.eoddata.com:80\nUser-Agent: Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.8.1.4) Gecko/20070603 Fedora/2.0.0.4-2.fc7 Firefox/2.0.0.4\nAccept: text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5\nAccept-Encoding: gzip,deflate\nAccept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7\nKeep-Alive: 300\nConnection: keep-alive"; descriptor = sock1.opensock(SERVER); ///////////////////////////////////////// // Socket Descriptors to read and write // from the server ///////////////////////////////////////// if(!(sockin = fdopen(descriptor, "r"))){ perror("cant open sockin file\n"); exit(1); } if(!(sockout = fdopen(descriptor, "w"))){ perror("cant open sockout file\n"); exit(1); } ///////////////////////////////////////// // send the URL and header info over the // socket ///////////////////////////////////////// fprintf(sockout,"%s\n%s\n",url,header); fflush(sockout); ///////////////////////////////////////// // Read in the HTML from the web server ///////////////////////////////////////// while(fgets(buf,MAXDATASIZE-1,sockin)){ fprintf(outfile,"%s",buf); memset(buf,0,0); } fclose(sockin); fclose(sockout); close(descriptor); } ////////////////////////////////////////////// // NASDEQ ////////////////////////////////////////////// void *getNASDAQ(void *){ FILE *out; if(!(out=fopen("ticFiles/NASDAQ-html","a+"))){ printf("can't open NASDAQ-html\n"); return NULL; } printf("Getting NASDAQ Symbols\n"); getTickers("GET /Symbols/NASDAQ/A.htm ",out); getTickers("GET /Symbols/NASDAQ/B.htm ",out); getTickers("GET /Symbols/NASDAQ/C.htm ",out); getTickers("GET /Symbols/NASDAQ/D.htm ",out); getTickers("GET /Symbols/NASDAQ/E.htm ",out); getTickers("GET /Symbols/NASDAQ/F.htm ",out); getTickers("GET /Symbols/NASDAQ/G.htm ",out); getTickers("GET /Symbols/NASDAQ/H.htm ",out); getTickers("GET /Symbols/NASDAQ/I.htm ",out); getTickers("GET /Symbols/NASDAQ/J.htm ",out); getTickers("GET /Symbols/NASDAQ/K.htm ",out); getTickers("GET /Symbols/NASDAQ/L.htm ",out); getTickers("GET /Symbols/NASDAQ/M.htm ",out); getTickers("GET /Symbols/NASDAQ/N.htm ",out); getTickers("GET /Symbols/NASDAQ/O.htm ",out); getTickers("GET /Symbols/NASDAQ/P.htm ",out); getTickers("GET /Symbols/NASDAQ/Q.htm ",out); getTickers("GET /Symbols/NASDAQ/R.htm ",out); getTickers("GET /Symbols/NASDAQ/S.htm ",out); getTickers("GET /Symbols/NASDAQ/T.htm ",out); getTickers("GET /Symbols/NASDAQ/U.htm ",out); getTickers("GET /Symbols/NASDAQ/V.htm ",out); getTickers("GET /Symbols/NASDAQ/W.htm ",out); getTickers("GET /Symbols/NASDAQ/X.htm ",out); getTickers("GET /Symbols/NASDAQ/Y.htm ",out); getTickers("GET /Symbols/NASDAQ/Z.htm ",out); fclose(out); return NULL; } ////////////////////////////////////////////// // American Stock Exchange ////////////////////////////////////////////// void *getAMEX(void *){ FILE *out; if(!(out=fopen("ticFiles/AMEX-html","a+"))){ printf("can't open AMEX-html\n"); return NULL; } printf("Getting AMEX Symbols\n"); getTickers("GET /Symbols/AMEX/A.htm ",out); getTickers("GET /Symbols/AMEX/B.htm ",out); getTickers("GET /Symbols/AMEX/C.htm ",out); getTickers("GET /Symbols/AMEX/D.htm ",out); getTickers("GET /Symbols/AMEX/E.htm ",out); getTickers("GET /Symbols/AMEX/F.htm ",out); getTickers("GET /Symbols/AMEX/G.htm ",out); getTickers("GET /Symbols/AMEX/H.htm ",out); getTickers("GET /Symbols/AMEX/I.htm ",out); getTickers("GET /Symbols/AMEX/J.htm ",out); getTickers("GET /Symbols/AMEX/K.htm ",out); getTickers("GET /Symbols/AMEX/L.htm ",out); getTickers("GET /Symbols/AMEX/M.htm ",out); getTickers("GET /Symbols/AMEX/N.htm ",out); getTickers("GET /Symbols/AMEX/O.htm ",out); getTickers("GET /Symbols/AMEX/P.htm ",out); getTickers("GET /Symbols/AMEX/Q.htm ",out); getTickers("GET /Symbols/AMEX/R.htm ",out); getTickers("GET /Symbols/AMEX/S.htm ",out); getTickers("GET /Symbols/AMEX/T.htm ",out); getTickers("GET /Symbols/AMEX/U.htm ",out); getTickers("GET /Symbols/AMEX/V.htm ",out); getTickers("GET /Symbols/AMEX/W.htm ",out); getTickers("GET /Symbols/AMEX/X.htm ",out); getTickers("GET /Symbols/AMEX/Y.htm ",out); getTickers("GET /Symbols/AMEX/Z.htm ",out); fclose(out); return NULL; } ////////////////////////////////////////////// // Chiago Board of Trade ////////////////////////////////////////////// void *getCBOT(void *){ FILE *out; if(!(out=fopen("ticFiles/CBOT-html","a+"))){ printf("can't open CBOT-html\n"); return NULL; } printf("Getting CBOT Symbols\n"); getTickers("GET /Symbols/CBOT/A.htm ",out); getTickers("GET /Symbols/CBOT/B.htm ",out); getTickers("GET /Symbols/CBOT/C.htm ",out); getTickers("GET /Symbols/CBOT/D.htm ",out); getTickers("GET /Symbols/CBOT/E.htm ",out); getTickers("GET /Symbols/CBOT/F.htm ",out); getTickers("GET /Symbols/CBOT/G.htm ",out); getTickers("GET /Symbols/CBOT/H.htm ",out); getTickers("GET /Symbols/CBOT/I.htm ",out); getTickers("GET /Symbols/CBOT/J.htm ",out); getTickers("GET /Symbols/CBOT/K.htm ",out); getTickers("GET /Symbols/CBOT/L.htm ",out); getTickers("GET /Symbols/CBOT/M.htm ",out); getTickers("GET /Symbols/CBOT/N.htm ",out); getTickers("GET /Symbols/CBOT/O.htm ",out); getTickers("GET /Symbols/CBOT/P.htm ",out); getTickers("GET /Symbols/CBOT/Q.htm ",out); getTickers("GET /Symbols/CBOT/R.htm ",out); getTickers("GET /Symbols/CBOT/S.htm ",out); getTickers("GET /Symbols/CBOT/T.htm ",out); getTickers("GET /Symbols/CBOT/U.htm ",out); getTickers("GET /Symbols/CBOT/V.htm ",out); getTickers("GET /Symbols/CBOT/W.htm ",out); getTickers("GET /Symbols/CBOT/X.htm ",out); getTickers("GET /Symbols/CBOT/Y.htm ",out); getTickers("GET /Symbols/CBOT/Z.htm ",out); fclose(out); return NULL; } ////////////////////////////////////////////// // Chiago Merchantile Exchange ////////////////////////////////////////////// void *getCME(void *){ FILE *out; if(!(out=fopen("ticFiles/CME-html","a+"))){ printf("can't open CME-html\n"); return NULL; } printf("Getting CME Symbols\n"); getTickers("GET /Symbols/CME/A.htm ",out); getTickers("GET /Symbols/CME/B.htm ",out); getTickers("GET /Symbols/CME/C.htm ",out); getTickers("GET /Symbols/CME/D.htm ",out); getTickers("GET /Symbols/CME/E.htm ",out); getTickers("GET /Symbols/CME/F.htm ",out); getTickers("GET /Symbols/CME/G.htm ",out); getTickers("GET /Symbols/CME/H.htm ",out); getTickers("GET /Symbols/CME/I.htm ",out); getTickers("GET /Symbols/CME/J.htm ",out); getTickers("GET /Symbols/CME/K.htm ",out); getTickers("GET /Symbols/CME/L.htm ",out); getTickers("GET /Symbols/CME/M.htm ",out); getTickers("GET /Symbols/CME/N.htm ",out); getTickers("GET /Symbols/CME/O.htm ",out); getTickers("GET /Symbols/CME/P.htm ",out); getTickers("GET /Symbols/CME/Q.htm ",out); getTickers("GET /Symbols/CME/R.htm ",out); getTickers("GET /Symbols/CME/S.htm ",out); getTickers("GET /Symbols/CME/T.htm ",out); getTickers("GET /Symbols/CME/U.htm ",out); getTickers("GET /Symbols/CME/V.htm ",out); getTickers("GET /Symbols/CME/W.htm ",out); getTickers("GET /Symbols/CME/X.htm ",out); getTickers("GET /Symbols/CME/Y.htm ",out); getTickers("GET /Symbols/CME/Z.htm ",out); fclose(out); return NULL; } ////////////////////////////////////////////// // Foreign Exchange ////////////////////////////////////////////// void *getFOREX(void *){ FILE *out; if(!(out=fopen("ticFiles/FOREX-html","a+"))){ printf("can't open FOREX-html\n"); return NULL; } printf("Getting FOREX Symbols\n"); getTickers("GET /Symbols/FOREX/A.htm ",out); getTickers("GET /Symbols/FOREX/B.htm ",out); getTickers("GET /Symbols/FOREX/C.htm ",out); getTickers("GET /Symbols/FOREX/D.htm ",out); getTickers("GET /Symbols/FOREX/E.htm ",out); getTickers("GET /Symbols/FOREX/F.htm ",out); getTickers("GET /Symbols/FOREX/G.htm ",out); getTickers("GET /Symbols/FOREX/H.htm ",out); getTickers("GET /Symbols/FOREX/I.htm ",out); getTickers("GET /Symbols/FOREX/J.htm ",out); getTickers("GET /Symbols/FOREX/K.htm ",out); getTickers("GET /Symbols/FOREX/L.htm ",out); getTickers("GET /Symbols/FOREX/M.htm ",out); getTickers("GET /Symbols/FOREX/N.htm ",out); getTickers("GET /Symbols/FOREX/O.htm ",out); getTickers("GET /Symbols/FOREX/P.htm ",out); getTickers("GET /Symbols/FOREX/Q.htm ",out); getTickers("GET /Symbols/FOREX/R.htm ",out); getTickers("GET /Symbols/FOREX/S.htm ",out); getTickers("GET /Symbols/FOREX/T.htm ",out); getTickers("GET /Symbols/FOREX/U.htm ",out); getTickers("GET /Symbols/FOREX/V.htm ",out); getTickers("GET /Symbols/FOREX/W.htm ",out); getTickers("GET /Symbols/FOREX/X.htm ",out); getTickers("GET /Symbols/FOREX/Y.htm ",out); getTickers("GET /Symbols/FOREX/Z.htm ",out); fclose(out); return NULL; } ////////////////////////////////////////////// // Global Indices ////////////////////////////////////////////// void *getINDEX(void *){ FILE *out; if(!(out=fopen("ticFiles/INDEX-html","a+"))){ printf("can't open INDEX-html\n"); return NULL; } printf("Getting INDEX Symbols\n"); getTickers("GET /Symbols/INDEX/A.htm ",out); getTickers("GET /Symbols/INDEX/B.htm ",out); getTickers("GET /Symbols/INDEX/C.htm ",out); getTickers("GET /Symbols/INDEX/D.htm ",out); getTickers("GET /Symbols/INDEX/E.htm ",out); getTickers("GET /Symbols/INDEX/F.htm ",out); getTickers("GET /Symbols/INDEX/G.htm ",out); getTickers("GET /Symbols/INDEX/H.htm ",out); getTickers("GET /Symbols/INDEX/I.htm ",out); getTickers("GET /Symbols/INDEX/J.htm ",out); getTickers("GET /Symbols/INDEX/K.htm ",out); getTickers("GET /Symbols/INDEX/L.htm ",out); getTickers("GET /Symbols/INDEX/M.htm ",out); getTickers("GET /Symbols/INDEX/N.htm ",out); getTickers("GET /Symbols/INDEX/O.htm ",out); getTickers("GET /Symbols/INDEX/P.htm ",out); getTickers("GET /Symbols/INDEX/Q.htm ",out); getTickers("GET /Symbols/INDEX/R.htm ",out); getTickers("GET /Symbols/INDEX/S.htm ",out); getTickers("GET /Symbols/INDEX/T.htm ",out); getTickers("GET /Symbols/INDEX/U.htm ",out); getTickers("GET /Symbols/INDEX/V.htm ",out); getTickers("GET /Symbols/INDEX/W.htm ",out); getTickers("GET /Symbols/INDEX/X.htm ",out); getTickers("GET /Symbols/INDEX/Y.htm ",out); getTickers("GET /Symbols/INDEX/Z.htm ",out); fclose(out); return NULL; } ////////////////////////////////////////////// // Hong Kong Stock Exchange ////////////////////////////////////////////// void *getHKSE(void *){ FILE *out; if(!(out=fopen("ticFiles/HKSE-html","a+"))){ printf("can't open HKSE-html\n"); return NULL; } printf("Getting HKSE Symbols\n"); getTickers("GET /Symbols/HKSE/0.htm ",out); getTickers("GET /Symbols/HKSE/1.htm ",out); getTickers("GET /Symbols/HKSE/2.htm ",out); getTickers("GET /Symbols/HKSE/3.htm ",out); getTickers("GET /Symbols/HKSE/4.htm ",out); getTickers("GET /Symbols/HKSE/5.htm ",out); getTickers("GET /Symbols/HKSE/6.htm ",out); getTickers("GET /Symbols/HKSE/7.htm ",out); getTickers("GET /Symbols/HKSE/8.htm ",out); fclose(out); return NULL; } ////////////////////////////////////////////// // London Stock Exchange ////////////////////////////////////////////// void *getLSE(void *){ FILE *out; if(!(out=fopen("ticFiles/LSE-html","a+"))){ printf("can't open LSE-html\n"); return NULL; } printf("Getting LSE Symbols\n"); getTickers("GET /Symbols/LSE/3.htm ",out); getTickers("GET /Symbols/LSE/4.htm ",out); getTickers("GET /Symbols/LSE/8.htm ",out); getTickers("GET /Symbols/LSE/A.htm ",out); getTickers("GET /Symbols/LSE/B.htm ",out); getTickers("GET /Symbols/LSE/C.htm ",out); getTickers("GET /Symbols/LSE/D.htm ",out); getTickers("GET /Symbols/LSE/E.htm ",out); getTickers("GET /Symbols/LSE/F.htm ",out); getTickers("GET /Symbols/LSE/G.htm ",out); getTickers("GET /Symbols/LSE/H.htm ",out); getTickers("GET /Symbols/LSE/I.htm ",out); getTickers("GET /Symbols/LSE/J.htm ",out); getTickers("GET /Symbols/LSE/K.htm ",out); getTickers("GET /Symbols/LSE/L.htm ",out); getTickers("GET /Symbols/LSE/M.htm ",out); getTickers("GET /Symbols/LSE/N.htm ",out); getTickers("GET /Symbols/LSE/O.htm ",out); getTickers("GET /Symbols/LSE/P.htm ",out); getTickers("GET /Symbols/LSE/Q.htm ",out); getTickers("GET /Symbols/LSE/R.htm ",out); getTickers("GET /Symbols/LSE/S.htm ",out); getTickers("GET /Symbols/LSE/T.htm ",out); getTickers("GET /Symbols/LSE/U.htm ",out); getTickers("GET /Symbols/LSE/V.htm ",out); getTickers("GET /Symbols/LSE/W.htm ",out); getTickers("GET /Symbols/LSE/X.htm ",out); getTickers("GET /Symbols/LSE/Y.htm ",out); getTickers("GET /Symbols/LSE/Z.htm ",out); fclose(out); return NULL; } ////////////////////////////////////////////// // Mutal Funds ////////////////////////////////////////////// void *getUSMF(void *){ FILE *out; if(!(out=fopen("ticFiles/USMF-html","a+"))){ printf("can't open USMF-html\n"); return NULL; } printf("Getting USMF Symbols\n"); getTickers("GET /Symbols/USMF/A.htm ",out); getTickers("GET /Symbols/USMF/B.htm ",out); getTickers("GET /Symbols/USMF/C.htm ",out); getTickers("GET /Symbols/USMF/D.htm ",out); getTickers("GET /Symbols/USMF/E.htm ",out); getTickers("GET /Symbols/USMF/F.htm ",out); getTickers("GET /Symbols/USMF/G.htm ",out); getTickers("GET /Symbols/USMF/H.htm ",out); getTickers("GET /Symbols/USMF/I.htm ",out); getTickers("GET /Symbols/USMF/J.htm ",out); getTickers("GET /Symbols/USMF/K.htm ",out); getTickers("GET /Symbols/USMF/L.htm ",out); getTickers("GET /Symbols/USMF/M.htm ",out); getTickers("GET /Symbols/USMF/N.htm ",out); getTickers("GET /Symbols/USMF/O.htm ",out); getTickers("GET /Symbols/USMF/P.htm ",out); getTickers("GET /Symbols/USMF/Q.htm ",out); getTickers("GET /Symbols/USMF/R.htm ",out); getTickers("GET /Symbols/USMF/S.htm ",out); getTickers("GET /Symbols/USMF/T.htm ",out); getTickers("GET /Symbols/USMF/U.htm ",out); getTickers("GET /Symbols/USMF/V.htm ",out); getTickers("GET /Symbols/USMF/W.htm ",out); getTickers("GET /Symbols/USMF/X.htm ",out); getTickers("GET /Symbols/USMF/Y.htm ",out); getTickers("GET /Symbols/USMF/Z.htm ",out); fclose(out); return NULL; } ////////////////////////////////////////////// // New York Stock Exchange ////////////////////////////////////////////// void *getNYSE(void *){ FILE *out; if(!(out=fopen("ticFiles/NYSE-html","a+"))){ printf("can't open NYSE-html\n"); return NULL; } printf("Getting NYSE Symbols\n"); getTickers("GET /Symbols/NYSE/A.htm ",out); getTickers("GET /Symbols/NYSE/B.htm ",out); getTickers("GET /Symbols/NYSE/C.htm ",out); getTickers("GET /Symbols/NYSE/D.htm ",out); getTickers("GET /Symbols/NYSE/E.htm ",out); getTickers("GET /Symbols/NYSE/F.htm ",out); getTickers("GET /Symbols/NYSE/G.htm ",out); getTickers("GET /Symbols/NYSE/H.htm ",out); getTickers("GET /Symbols/NYSE/I.htm ",out); getTickers("GET /Symbols/NYSE/J.htm ",out); getTickers("GET /Symbols/NYSE/K.htm ",out); getTickers("GET /Symbols/NYSE/L.htm ",out); getTickers("GET /Symbols/NYSE/M.htm ",out); getTickers("GET /Symbols/NYSE/N.htm ",out); getTickers("GET /Symbols/NYSE/O.htm ",out); getTickers("GET /Symbols/NYSE/P.htm ",out); getTickers("GET /Symbols/NYSE/Q.htm ",out); getTickers("GET /Symbols/NYSE/R.htm ",out); getTickers("GET /Symbols/NYSE/S.htm ",out); getTickers("GET /Symbols/NYSE/T.htm ",out); getTickers("GET /Symbols/NYSE/U.htm ",out); getTickers("GET /Symbols/NYSE/V.htm ",out); getTickers("GET /Symbols/NYSE/W.htm ",out); getTickers("GET /Symbols/NYSE/X.htm ",out); getTickers("GET /Symbols/NYSE/Y.htm ",out); getTickers("GET /Symbols/NYSE/Z.htm ",out); fclose(out); return NULL; } ///////////////////////////// // driver program ///////////////////////////// int main(int argc, char *argv[]){ pthread_t ticNYSE; pthread_t ticLSE; pthread_t ticUSMF; pthread_t ticHKSE; pthread_t ticINDEX; pthread_t ticFOREX; pthread_t ticCME; pthread_t ticCBOT; pthread_t ticAMEX; pthread_t ticNASDAQ; if((pthread_create( &ticNYSE,NULL,getNYSE,NULL))!= 0){ printf("can't create new thread .. exiting\n"); exit(1); } if((pthread_create( &ticLSE,NULL,getLSE,NULL))!= 0){ printf("can't create new thread .. exiting\n"); exit(1); } if((pthread_create( &ticUSMF,NULL,getUSMF,NULL))!= 0){ printf("can't create new thread .. exiting\n"); exit(1); } if((pthread_create( &ticHKSE,NULL,getHKSE,NULL))!= 0){ printf("can't create new thread .. exiting\n"); exit(1); } if((pthread_create( &ticINDEX,NULL,getINDEX,NULL))!= 0){ printf("can't create new thread .. exiting\n"); exit(1); } if((pthread_create( &ticFOREX,NULL,getFOREX,NULL))!= 0){ printf("can't create new thread .. exiting\n"); exit(1); } if((pthread_create( &ticCBOT,NULL,getCBOT,NULL))!= 0){ printf("can't create new thread .. exiting\n"); exit(1); } if((pthread_create( &ticAMEX,NULL,getAMEX,NULL))!= 0){ printf("can't create new thread .. exiting\n"); exit(1); } if((pthread_create( &ticCME,NULL,getCME,NULL))!= 0){ printf("can't create new thread .. exiting\n"); exit(1); } if((pthread_create( &ticNASDAQ,NULL,getNASDAQ,NULL))!= 0){ printf("can't create new thread .. exiting\n"); exit(1); } pthread_join( ticNASDAQ,NULL); pthread_join( ticCME,NULL); pthread_join( ticAMEX,NULL); pthread_join( ticCBOT,NULL); pthread_join( ticFOREX,NULL); pthread_join( ticINDEX,NULL); pthread_join( ticHKSE,NULL); pthread_join( ticUSMF,NULL); pthread_join( ticLSE,NULL); pthread_join( ticNYSE,NULL); exit(0); }