#!/usr/pkg/bin/rexx
/*-----------------------------------------------------------------------------+
| Program name:                                                                |
|                                                                              |
| Subject: Download xBSD application index files                               |
|                                                                              |
| File name: bsd_wget_index_files.rexx                                         |
|                                                                              |
| Note:                                                                        |
|                                                                              |
| Category: Freeware, BSD Licenced                                             |
|                                                                              |
| Copyright (C) by Przemysław Pawełczyk                       Cracow, May 2007 |
| E-Mail:                                                          pp_o2@o2.pl |
| P2O2 WWW:                                                http://pp.kv.net.pl |
+-----------------------------------------------------------------------------*/
/*
################################################################################
#
# 3 index files for FreeBSD, NetBSD, OpenBSD can be downloaded
# by bsd_wget_index_files.sh as well
#
# ==> describes hyperlinks from which the files were downloaded
#
# FreeBSD
# ftp://ftp.freebsd.org/pub/FreeBSD/ports/i386/packages-stable/All/
# ===> fbsd=http://www.freebsd.org/ports/master-index.html
#
# NetBSD
# ftp://ftp.NetBSD.org/pub/pkgsrc/packages/NetBSD-3.1/i386/All/0verkill-0.16.tgz
# ftp://ftp.netbsd.org/pub/pkgsrc/packages-2007Q1/NetBSD-3.1/i386/All/
# ===> nbsd=ftp://ftp.NetBSD.org/pub/NetBSD/packages/pkgsrc/README-all.html
#
# OpenBSD
# ftp://ftp.openbsd.org/pub/OpenBSD/4.1/packages/i386/915resolution-0.5.2.tgz
# ftp://ftp.task.gda.pl/pub/OpenBSD/4.1/packages/i386/
# ===> obsd=http://www.openbsd.org/4.1_packages/i386.html
#
################################################################################
*/

/*-----------------------------------------------------------------------------+
| Setting variables                                                            |
+-----------------------------------------------------------------------------*/
/*---------------------------------------------------------+
| Setting index file dir and reading current directory and |
+---------------------------------------------------------*/
homPath = "/home/lnbsd"
bsdPath = homPath"/lnbsd/dict.db/dict.bsd/"

/*---------------------------------------------------------+
| BSD names                                                |
+---------------------------------------------------------*/
bsdName    = .array~new(3)
bsdName[1] = "FreeBSD"
bsdName[2] = "NetBSD"
bsdName[3] = "OpenBSD"

/*---------------------------------------------------------+
| BSD renamed index files                                  |
+---------------------------------------------------------*/
bsdFile    = .array~new(3)
bsdFile[1] = "fbsd_index.html"
bsdFile[2] = "nbsd_index.html"
bsdFile[3] = "obsd_index.html"

/*---------------------------------------------------------+
| BSD new (procsessed from html) index files               |
+---------------------------------------------------------*/
bsdText    = .array~new(3)
bsdText[1] = "fbsd_index.txt"
bsdText[2] = "nbsd_index.txt"
bsdText[3] = "obsd_index.txt"

/*---------------------------------------------------------+
| BSD index files to download                              |
+---------------------------------------------------------*/
bsdLoad    = .array~new( 3 )
bsdLoad[1] = "http://www.freebsd.org/ports/master-index.html"
bsdLoad[2] = "ftp://ftp.NetBSD.org/pub/NetBSD/packages/pkgsrc/README-all.html"
bsdLoad[3] = "http://www.openbsd.org/4.1_packages/i386.html"

/*---------------------------------------------------------+
| BSD ftp addresses                                        |
+---------------------------------------------------------*/
bsdFtp    = .array~new( 3 )
bsdFtp[1] = "ftp://ftp.freebsd.org/pub/FreeBSD/ports/i386/packages-stable/All/"
bsdFtp[2] = "ftp://ftp.netbsd.org/pub/pkgsrc/packages-2007Q1/NetBSD-3.1/i386/All/"
bsdFtp[3] = "ftp://ftp.openbsd.org/pub/OpenBSD/4.1/packages/i386/"

/*---------------------------------------------------------+
| BSD control variables                                    |
+---------------------------------------------------------*/
bsdIndx = .array~of( 0, 0, 0 )

PkgLen = 40               /* Application name span length displayed on screen */

/*-----------------------------------------------------------------------------+
|  >>>>>>>>>>>>>>>>>>>>>>>>>  Main program body <<<<<<<<<<<<<<<<<<<<<<<<<<<<<  |
+-----------------------------------------------------------------------------*/
call header

/*-----------------------------------------------------------------------------+
|  Downloading BSD "native" index files                                        |
+-----------------------------------------------------------------------------*/
do i = 1 to bsdLoad~items

/*---------------------------------------------------------+
| Checking if renamed index files exist                    |
+---------------------------------------------------------*/
  bsdTemp = bsdPath''bsdFile[i]
  if bsdTemp = stream( bsdTemp, "c", "query exists" ) then do
    bsdIndx[i] = 1
    say bsdName[i]" "bsdFile[i]" file exists"
    say
  end
  else do
/*---------------------------------------------------------+
| Download file                                            |
+---------------------------------------------------------*/
    'wget -c -nc -v -P 'bsdPath' 'bsdLoad[i]
  end  /*-- if bsdLoad[i] --*/

end  /*-- do i = 1 --*/


/*-----------------------------------------------------------------------------+
| Renaming downloaded index files                                              |
+-----------------------------------------------------------------------------*/
do i = 1 to bsdLoad~items

/*---------------------------------------------------------+
| Cutting index file name                                  |
+---------------------------------------------------------*/
  LoadName = filespec( "N", bsdLoad[i] )

/*---------------------------------------------------------+
| Checking if index files are downloaded                   |
+---------------------------------------------------------*/
  bsdTemp = bsdPath''LoadName
  if bsdTemp = stream( bsdTemp, "c", "query exists" ) then do

    say 'Changing 'bsdName[i]' index file to "'bsdFile[i]'"'

/*---------------------------------------------------------+
|  Reading File to Variable                                |
+---------------------------------------------------------*/
    Call Stream bsdTemp, 'c', 'open read'
    TempFile  = charin( bsdTemp, 1, chars( bsdTemp ) )
    Call Stream bsdTemp, 'c', 'close'

/*---------------------------------------------------------+
| Renaming downloaded index file                           |
+---------------------------------------------------------*/
    Call SysFileDelete( bsdTemp )

/*---------------------------------------------------------+
|  Writing Variable to File                                |
+---------------------------------------------------------*/
    bsdTemp = bsdPath''bsdFile[i]
    Call Stream bsdTemp, 'c', 'open write'
    ok = charout( bsdTemp, TempFile )
    Call Stream bsdTemp, 'c', 'close'

/*
    'mv 'bsdPath''LoadName' 'bsdPath''bsdFile[i]
*/
/*---------------------------------------------------------+
| Making copy of renamed index file                        |
+---------------------------------------------------------*/
    Call Stream bsdTemp".old", 'c', 'open write'
    ok = charout( bsdTemp".old", TempFile )
    Call Stream bsdTemp".old", 'c', 'close'

/*
    'cp 'bsdPath''bsdFile[i]' 'bsdPath''bsdFile[i]'.old'
*/

  end
  else do
/*---------------------------------------------------------+
| No native and renamed index files exist                  |
+---------------------------------------------------------*/
    if bsdIndx[i] = 0 then do
      say "No "bsdName[i]" "LoadName" file found"
    end

  end  /*-- if LoadName~query --*/

end  /*-- do i = 1 --*/

drop bsdTemp
say ""


call bsd_new_indexes


/*---------------------------------------------------------+
| Printing ending sequence                                 |
+---------------------------------------------------------*/
call footer
/*-----------------------------------------------------------------------------+
|  >>>>>>>>>>>>>>>>>>>>>> End of Main program body <<<<<<<<<<<<<<<<<<<<<<<<<<  |
+-----------------------------------------------------------------------------*/
exit

/*-----------------------------------------------------------------------------+
|  PROCEDURES AND FUNCTIONS                                                    |
+-----------------------------------------------------------------------------*/
header:

  say
  say "=========================================================="
  say "=                                                        ="
  say "=       ...running bsd_wget_index_files.rexx...          ="
  say "=                                                        ="
  say "=========================================================="
  say

return

footer:

  say
  say "=========================================================="
  say "=                                                        ="
  say "=    ...END of running bsd_wget_index_files.rexx...      ="
  say "=                                                        ="
  say "=========================================================="
  say
  
return

bsd_new_indexes:
/*---------------------------------------------------------+
| Loop over all 3 BSD HTML index files                     |
+---------------------------------------------------------*/
  do i = 1 to bsdFile~items

    WriteStem. = ""

    File2Read  = bsdPath''bsdFile[i]
    File2Write = bsdPath''bsdText[i]

    bsd2Read  = File2Read
    bsd2Write = File2Write

/*---------------------------------------------------------+
| Reading xbsd_index.html files                            |
+---------------------------------------------------------*/
    ok = stream( File2Read, 'c','open read' )  /* Opening File2Read           */
    j = 1
    do while lines( File2Read ) > 0            /* If File2Read contains lines */
      TempLine = linein( File2Read )           /* then read them in FileStem  */

/*---------------------------------------------------------+
| Parsing xbsd_index.html files                            |
+---------------------------------------------------------*/
      if i = 1 then do
        PosNumber = pos( "<STRONG><A HREF", TempLine )
        if ( PosNumber >= 1 ) & ( PosNumber < 5 ) then do
          parse var TempLine _s1 '">' FileStr '</A>'_s2'<EM>'FileDes'</EM>'_s3
          FileStr = strip( FileStr )
          FileDes = strip( FileDes )
          WriteStem.j = left( FileStr, PkgLen )""FileDes
        end
        else iterate
      end

      if i = 2 then do
        PosNumber = pos( "(for sorting)", TempLine )
        if ( PosNumber \= 0 ) then do
          parse var TempLine _s1 '<!--' FileStr '(for'_s2'<td>'FileDes
          FileStr = strip( FileStr )
          FileDes = strip( FileDes )
          WriteStem.j = left( FileStr, PkgLen )""FileDes
        end
        else iterate
      end  /*-- if i = 2 --*/

      if i = 3 then do
        PosNumber = pos( "i386", TempLine )
        if ( PosNumber = 16 ) then do
          parse var TempLine _s1 'html>' FileStr '</a>'
          FileStr = strip( FileStr )

          NextTempLine = linein( File2Read )   /* then read them in FileStem  */
          parse var NextTempLine _s1 '<i>' FileDes '</i>' _s2
          FileDes = strip( FileDes )

          WriteStem.j = left( FileStr, PkgLen )""FileDes
        end
        else iterate
      end  /*-- if i = 3 --*/

      j = j + 1
    end /* do while */

    WriteStem.0 = j - 1                        /* Number of lines copied      */
    ok = stream(File2Read, 'c','close')        /* Closing File2Read           */

/*---------------------------------------------------------+
| Checking if index.txt files exist                        |
+---------------------------------------------------------*/
    bsdTemp = bsd2Write
    if bsdTemp = stream( bsdTemp, "c", "query exists" ) then do

      say 'Making copy of 'bsdText[i]' file to "'bsdText[i]'.old"'

/*---------------------------------------------------------+
|  Reading File to Variable                                |
+---------------------------------------------------------*/
      call stream bsdTemp, 'c', 'open read'
      TempFile  = charin( bsdTemp, 1, chars( bsdTemp ) )
      call stream bsdTemp, 'c', 'close'

/*---------------------------------------------------------+
| Making copy of renamed index file                        |
+---------------------------------------------------------*/
      call stream bsdTemp".old", 'c', 'open write'
      ok = charout( bsdTemp".old", TempFile )
      call stream bsdTemp".old", 'c', 'close'

    end  /*-- if bsdTemp =  --*/

/*---------------------------------------------------------+
| Deleting xbsd_index.txt files                            |
+---------------------------------------------------------*/
    call SysFileDelete( bsdTemp )

/*---------------------------------------------------------+
| Making xbsd_index.txt files                              |
+---------------------------------------------------------*/
    ok = stream(  File2Write, 'c','open write' ) /* Opening File2Write        */
    ok = lineout( File2Write, "")
    ok = lineout( File2Write, bsdFtp[i])
    ok = lineout( File2Write, "")
    do j = 1 to WriteStem.0
      ok = lineout( File2Write, WriteStem.j )
    end
    ok = stream( File2Write, 'c','close' )     /* Closing File2Write          */

  end

return