/* --------------------------------------------------------------- */
/* Retrieve the data associated with a URL (e.g., a web page).     */
/* --------------------------------------------------------------- */
/*                                                                 */
/* Copyright (c) Mike Cowlishaw, 1994-2012.  All rights reserved.  */
/* Parts Copyright (c) IBM, 1994-2009.                             */
/*                                                                 */
/* Permission to use, copy, modify, and distribute this software   */
/* for any non-commercial purpose without fee is hereby granted,   */
/* provided that the above copyright notice and this permission    */
/* notice appear in all copies, and that notice and the date of    */
/* any modifications be added to the software.                     */
/*                                                                 */
/* This software is provided "as is".  No warranties, whether      */
/* express, implied, or statutory, including, but not limited to,  */
/* implied warranties of merchantability and fitness for a         */
/* particular purpose apply to this software.  The author shall    */
/* not, in any circumstances, be liable for special, incidental,   */
/* or consequential damages, for any reason whatsoever.            */
/*                                                                 */
/* --------------------------------------------------------------- */
/* Argument words:                                                 */
/*                                                                 */
/*   url   -- URL to read                                          */
/*   proxy -- optional URL of proxy to use via                     */
/*                                                                 */
/*     where url does not need leading 'http://'                   */
/*     and proxy, if given, should be just proxy address           */
/*                                                                 */
/*     e.g:  geturl speleotrove.com wwwproxy                       */
/*                                                                 */
/*   returns the body of the URL if called as a function,          */
/*   or saves as gothead.$$$ and gotbody.$$$ if a command call.    */
/*                                                                 */
/* --------------------------------------------------------------- */
/* 1994.11.06 Initial version                                      */
/* 1999.04.24 Show stats if called as command                      */
/* 2002.01.14 Return body if not called as command                 */
/*            [empty if error]                                     */
/* 2004.11.30 Add Host: for fake "HTTP 1.1" servers                */
/* 2009.03.28 Use temporary file for accumulation to avoid         */
/*            system resources exhausted problem                   */
/* 2012.05.24 Cosmetic update                                      */

if RxFuncQuery("SockLoadFuncs") then do
  rc=RxFuncAdd("SockLoadFuncs","rxSock","SockLoadFuncs")
  rc=SockLoadFuncs()
  end

parse arg url proxy .

if left(url,7)='http://' then url=substr(url,8)
parse var url server '/' what
if server='' then server='speleotrove.com'
what='/'what                  /* what to ask for */
if proxy<>'' then do
  what='http://'server||what
  server=proxy
  end

crlf    ='0d0a'x                             /* constants */
family  ='AF_INET'
parse var server server ':' httpport
if httpport='' then httpport=80

if verify(server, '.0123456789')=0 then dotserver=server
 else do
  rc=sockgethostbyname(server, "serv.0")     /* get dotaddress of server */
  if rc=0 then do; say 'Unable to resolve "'server'"'; exit ''; end
  dotserver=serv.0addr                       /* .. */
  end

gosaddr.0family=family                       /* set up address */
gosaddr.0port  =httpport
gosaddr.0addr  =dotserver

gosock = SockSocket(family, "SOCK_STREAM", "IPPROTO_TCP")

message = "GET" what 'HTTP/1.0'crlf
message = message'Accept-Language: en'crlf
message = message'Host:' server':'httpport''crlf
message = message'Pragma: no-cache'crlf
message = message'Cache-Control: no-cache'crlf
message = message''crlf

call time 'r'
rc = SockConnect(gosock,"gosaddr.0")
if rc<0 then do
  say 'Unable to connect to "'server':'httpport'"' gosaddr.0addr
  exit ''; end
conn=time('e')
rc = SockSend(gosock, message)

tfile='$temp$.$$$'
call qerase tfile
sent=time('e'); rtime.=sent
do r=1 by 1
  rc = SockRecv(gosock, "response", 1000) /*; say '>'rc'>' response */
  rtime.r=time('e')
  if rc<=0 then leave
  call charout tfile, response
  end
rlast=r-1
rc = SockClose(gosock)
call charout tfile

got=charin(tfile, 1, chars(tfile))
call charout tfile
call qerase tfile
-- say length(got) chars(tfile)


parse source . how .
if how='COMMAND' then do
  con=format(conn                ,,3)
  sen=format(sent-conn           ,,3)
  res=format(rtime.1-sent        ,,3)
  rec=format(rtime.rlast-rtime.1 ,,3)

  sendlen=length(message)
  readlen=length(got)

  if rlast=0 then red=' (Read none)'
             else red=' Read['readlen']:' rec
  say 'Conn:' con ' Send['sendlen']:' sen ' Wait:' res red
  end

/* Allow non-conforming headers */
lflf=pos('0a0a'x, got)
clcl=pos('0d0a0d0a'x, got)


if left(got, 5)\='HTTP/' then do   /* no header */
  header=''
  body=got
  end
 else if clcl=0 | (lflf<clcl & lflf>0) then parse var got header '0a0a'x body
 else parse var got header '0d0a0d0a'x body

call qerase 'gothead.$$$'
call qerase 'gotbody.$$$'


if how='COMMAND' then do
  call lineout 'gothead.$$$', header
  call lineout 'gothead.$$$'

  call lineout 'gotbody.$$$', body
  call lineout 'gotbody.$$$'
  exit
  end

return body

/* --------------------------------------------------------------- */
/* Quiet erase                                                     */
/* --------------------------------------------------------------- */
qerase: procedure
 parse arg spec
 call sysfiledelete spec
 return