: uls-master.bat : batch file to automate the entire process of managing of the FCC ULS (Master) files : n6lhv@arrl.net (Wayne Smith) : created: 28-July-2004 : updated: 30-Mar-2008 : functional dependencies : -ensure that the database server (e.g., mysqld) is running : ALWAYS do this processing, regardless of desired other functionality SET RPATH="e:\program files\r\R-2.6.2\bin\rterm" net stop MySQL del e:\mysql\.log net start MySQL mysqlcheck -v uls --analyze --optimize :cls :GOTO END GOTO DOWNLOAD :GOTO EXTRACT :GOTO PREPROCESS :GOTO CREATETABLES :GOTO IMPORTDATA :GOTO FREQUENCYDISTRIBUTIONS :GOTO MINORFIXES :GOTO IMPUTELOCATIONDATA :GOTO CREATEINDEXESBEFOREDELETE :GOTO ADDITIONALDISTRIBUTIONS :GOTO JOINUS :GOTO DELETENONCA :GOTO CREATEINDEXESAFTERDELETE :GOTO JOINCA :GOTO CREATEINDEXESCA :GOTO CREATEGEOTABLES :GOTO CREATEORGTABLES :GOTO CREATESAMPLETABLES :GOTO CREATETIMESERIES :GOTO PLOTTIMESERIES :GOTO SUMMARYSTATISTICS :GOTO ANALYZEDIVERSITY :GOTO ANALYZEDATAMINING :GOTO ANALYZESOCIALNETWORK :GOTO EXPORTMYSQL :GOTO EXPORTSQLITE :GOTO EXPORTMSACCESS :GOTO EXPORTDBASE :GOTO EXPORTMSEXCEL :GOTO EXPORTFIXEDWIDTH :GOTO GENERATESCHEMAS :GOTO DERIVEFILESIZES :GOTO UPLOAD :DOWNLOAD @call download.bat l_LMpriv @call download.bat l_LMcomm @call download.bat l_LMbcast @call download.bat l_coast @call download.bat l_micro @call download.bat l_paging date /t >download-date.txt :EXTRACT @call extract-pkzip.bat l_LMpriv @call extract-pkzip.bat l_LMcomm @call extract-pkzip.bat l_LMbcast @call extract-pkzip.bat l_coast @call extract-pkzip.bat l_micro @call extract-pkzip.bat l_paging :PREPROCESS @call preprocess-DeleteBlankLines.bat l_LMpriv-cp @call preprocess-DeleteBlankLines.bat l_LMpriv-en @call preprocess-DeleteBlankLines.bat l_LMpriv-lo @call preprocess-DeleteBlankLines.bat l_LMcomm-cp @call preprocess-DeleteBlankLines.bat l_LMcomm-en @call preprocess-DeleteBlankLines.bat l_LMcomm-lo @call preprocess-DeleteBlankLines.bat l_LMbcast-cp @call preprocess-DeleteBlankLines.bat l_LMbcast-en @call preprocess-DeleteBlankLines.bat l_LMbcast-lo @call preprocess-DeleteBlankLines.bat l_coast-cp @call preprocess-DeleteBlankLines.bat l_coast-en @call preprocess-DeleteBlankLines.bat l_coast-lo @call preprocess-DeleteBlankLines.bat l_micro-cp @call preprocess-DeleteBlankLines.bat l_micro-en @call preprocess-DeleteBlankLines.bat l_micro-lo @call preprocess-DeleteBlankLines.bat l_paging-cp @call preprocess-DeleteBlankLines.bat l_paging-en @call preprocess-DeleteBlankLines.bat l_paging-lo copy /y preprocess-DeleteBlankLines.pl preprocess-DeleteBlankLines-pl.txt @call preprocess-FixEmbeddedCRLFs.bat l_LMpriv-cp @call preprocess-FixEmbeddedCRLFs.bat l_LMpriv-en @call preprocess-FixEmbeddedCRLFs.bat l_LMpriv-lo @call preprocess-FixEmbeddedCRLFs.bat l_LMcomm-cp @call preprocess-FixEmbeddedCRLFs.bat l_LMcomm-en @call preprocess-FixEmbeddedCRLFs.bat l_LMcomm-lo @call preprocess-FixEmbeddedCRLFs.bat l_LMbcast-cp @call preprocess-FixEmbeddedCRLFs.bat l_LMbcast-en @call preprocess-FixEmbeddedCRLFs.bat l_LMbcast-lo @call preprocess-FixEmbeddedCRLFs.bat l_coast-cp @call preprocess-FixEmbeddedCRLFs.bat l_coast-en @call preprocess-FixEmbeddedCRLFs.bat l_coast-lo @call preprocess-FixEmbeddedCRLFs.bat l_micro-cp @call preprocess-FixEmbeddedCRLFs.bat l_micro-en @call preprocess-FixEmbeddedCRLFs.bat l_micro-lo @call preprocess-FixEmbeddedCRLFs.bat l_paging-cp @call preprocess-FixEmbeddedCRLFs.bat l_paging-en @call preprocess-FixEmbeddedCRLFs.bat l_paging-lo copy /y preprocess-FixEmbeddedCRLFs.pl preprocess-FixEmbeddedCRLFs-pl.txt @call preprocess-AddDB_IDField.bat l_LMpriv-fr @call preprocess-AddDB_IDField.bat l_LMcomm-fr @call preprocess-AddDB_IDField.bat l_LMbcast-fr @call preprocess-AddDB_IDField.bat l_coast-fr @call preprocess-AddDB_IDField.bat l_micro-fr @call preprocess-AddDB_IDField.bat l_paging-fr @call preprocess-AddDB_IDField.bat l_LMpriv-hs @call preprocess-AddDB_IDField.bat l_LMcomm-hs @call preprocess-AddDB_IDField.bat l_LMbcast-hs @call preprocess-AddDB_IDField.bat l_micro-hs @call preprocess-AddDB_IDField.bat l_paging-hs copy /y preprocess-AddDB_IDField.pl preprocess-AddDB_IDField-pl.txt :CREATETABLES mysql -vvv createdbandtables-mysql.log :IMPORTDATA mysql -vvv import-l_LMpriv-mysql-50x.log mysql -vvv import-l_LMcomm-mysql-50x.log mysql -vvv import-l_LMbcast-mysql-40x.log mysql -vvv import-l_coast-mysql-50x.log mysql -vvv import-l_micro-mysql-50x.log mysql -vvv import-l_paging-mysql-50x.log :FREQUENCYDISTRIBUTIONS mysqlcheck -v uls --analyze --optimize :mysql -vvv qryFreqDist.log :MINORFIXES mysqlcheck -v uls --analyze --optimize mysql -vvv qryFixTrailingCRLF.log :mysql -vvv qryPrependZerosForLowBand.log mysql -vvv qryFixLowerCase.log mysql -vvv qryEOBlankStates.log mysql -vvv qryEOBlankCounties.log mysql -vvv qryEOBlankCities.log :IMPUTELOCATIONDATA mysqlcheck -v uls --analyze --optimize mysql -vvv qryImputeCityFromAddress.log mysql -vvv qryImputeCountyFromCity.log mysql -vvv qryImputeStateFromCounty.log :CREATEINDEXESBEFOREDELETE mysqlcheck -v uls --analyze --optimize :mysql -vvv createindexes-mysql.log :ADDITIONALDISTRIBUTIONS mysqlcheck -v uls --analyze --optimize :mysql -vvv qryDistLocationStateCounty.log :mysql -vvv qryDistLocationStateCountyCity.log :JOINUS mysqlcheck -v uls --analyze --optimize :: this query is costly...it is not needed for most applications mysql -vvv qryGeoUS-Explain.log :mysql -vvv qryGeoUS.log ::mysql -vvv qryGeoUSBlanksOnly.log :DELETENONCA mysqlcheck -v uls --analyze --optimize mysql -vvv qryDeleteFreqsAbove1GHz.log mysql -vvv qryDeleteLocationNotStateCAOrBlank.log mysql -vvv qryDeleteControlNotStateCA.log mysql -vvv qryDeleteCellular.log ::mysql -vvv qryDeleteLocationNotBlank.log :CREATEINDEXESAFTERDELETE mysqlcheck -v uls --analyze --optimize mysql -vvv createindexes-mysql.log :JOINCA mysql -vvv qryGeoStateCA-pre.log mysql -vvv qryGeoStateCA-Analyze.log mysql -vvv qryGeoStateCA-Explain.log mysql -vvv qryGeoStateCA.log mysql -vvv qryGeoStateCA-post.log :CREATEINDEXESCA mysqlcheck -v uls --analyze --optimize mysql -vvv createindexes-stateCA-mysql.log :CREATEGEOTABLES mysql -vvv qryGeoCountiesSoCal.log mysql -vvv qryGeoCountyLosAngeles.log mysql -vvv qryGeoCityLosAngeles.log :mysql -vvv qryGeoCountiesMetroScan.log :mysql -vvv qryGeoCountiesPoliceCall.log :mysql -vvv qryGeoCountiesSCAN.log :mysql -vvv qryGeoCountiesScanFan.log :@call export-msaccess.bat tblGeoCountiesSoCalUnivariate :CREATEORGTABLES mysql -vvv qryOrgStateCA.log mysql -vvv qryOrgCountyImperial.log mysql -vvv qryOrgCountyKern.log mysql -vvv qryOrgCountyLosAngeles.log mysql -vvv qryOrgCountyOrange.log mysql -vvv qryOrgCountyRiverside.log mysql -vvv qryOrgCountySanBernardino.log mysql -vvv qryOrgCountySanDiego.log mysql -vvv qryOrgCountySanLuisObispo.log mysql -vvv qryOrgCountySantaBarbara.log mysql -vvv qryOrgCountyVentura.log mysql -vvv qryOrgCountyLosAngelesMTA.log mysql -vvv qryOrgCityLosAngeles.log mysql -vvv qryOrgCityLosAngelesWP.log mysql -vvv qryOrgAnchor.log mysql -vvv qryOrgBigRetail.log mysql -vvv qryOrgLargeSMR.log mysql -vvv qryOrgCPRA.log :CREATESAMPLETABLES mysql -vvv qrySampleLASDSimplex.log mysql -vvv qrySampleFederal.log mysql -vvv qrySampleRailroad.log :mysql -vvv qryEmulatePoliceCall.log :mysql -vvv qryStatBivariateCallsignByFrequency.log :mysql -vvv qryStatBivariateServiceByFrequency.log :mysql -vvv qryStatBivariateStationByFrequency.log :mysql -vvv qryStatBivariateEntityByFrequency.log :mysql -vvv qryStatBivariateFrequencyByCallsign.log :mysql -vvv qryStatBivariateServiceByCallsign.log :mysql -vvv qryStatBivariateStationByCallsign.log :mysql -vvv qryStatBivariateEntityByCallsign.log :mysql -vvv qryStatBivariateFrequencyByService.log :mysql -vvv qryStatBivariateCallsignByService.log :mysql -vvv qryStatBivariateStationByService.log :mysql -vvv qryStatBivariateEntityByService.log :mysql -vvv qryStatBivariateFrequencyByStation.log :mysql -vvv qryStatBivariateCallsignByStation.log :mysql -vvv qryStatBivariateServiceByStation.log :mysql -vvv qryStatBivariateEntityByStation.log :mysql -vvv qryStatBivariateFrequencyByEntity.log :mysql -vvv qryStatBivariateCallsignByEntity.log :mysql -vvv qryStatBivariateServiceByEntity.log :mysql -vvv qryStatBivariateStationByEntity.log :CREATETIMESERIES mysql -vvv qryTimeSeries1.log mysql -vvv qryTimeSeries2.log mysql -vvv qryTimeSeries3.log mysql -vvv qryTimeSeries4.log mysql -vvv qryTimeSeries5.log mysql -vvv qryTimeSeries6.log :PLOTTIMESERIES del liass-call_sign.png del liass-call_sign.pdf del liass-call_sign.svg del liren-call_sign.png del liren-call_sign.pdf del liren-call_sign.svg del liass-call_sign-scaled.png del liass-call_sign-scaled.pdf del liass-call_sign-scaled.svg del liren-call_sign-scaled.png del liren-call_sign-scaled.pdf del liren-call_sign-scaled.svg :%RPATH% --no-save plot.log %RPATH% --no-save plot.log :mysql -vvv qryPlotYear.log :mysql -vvv qryPlotTest-Explain.log :mysql -vvv qryPlotTest.log :mysql -vvv createindexes-plottest-mysql.log :@call export-plot.bat qryPlotGeoYearFrequencyAll :@call export-plot.bat qryPlotGeoYearFrequencyUnique :@call export-plot.bat qryPlotGeoYearFrequencyProportion :@call export-plot.bat qryPlotGeoYearFrequencyNew :@call export-plot.bat qryPlotGeoYearFrequency026050 :@call export-plot.bat qryPlotGeoYearFrequency150162 :@call export-plot.bat qryPlotGeoYearFrequency450512 :@call export-plot.bat qryPlotGeoYearFrequency806857 :@call export-plot.bat qryPlotGeoYearFrequency896940 :@call export-plot.bat qryPlotGeoYearFrequencyNew026050 :@call export-plot.bat qryPlotGeoYearFrequencyNew150162 :@call export-plot.bat qryPlotGeoYearFrequencyNew450512 :@call export-plot.bat qryPlotGeoYearFrequencyNew806857 :@call export-plot.bat qryPlotGeoYearFrequencyNew896940 :@call export-plot.bat qryPlotGeoYearCallsignAll :@call export-plot.bat qryPlotGeoYearCallsignUnique :@call export-plot.bat qryPlotGeoYearCallsignProportion :@call export-plot.bat qryPlotGeoYearCallsignEQ464575 :@call export-plot.bat qryPlotGeoYearEntityAll :@call export-plot.bat qryPlotGeoYearEntityUnique :@call export-plot.bat qryPlotGeoYearEntityProportion :@call export-plot.bat qryPlotGeoYearCallsignFrequency :@call export-plot.bat qryPlotGeoYearCallsignNewFrequency :@call export-plot.bat qryPlotGeoYearFrequencyCallsign :SUMMARYSTATISTICS mysqlcheck -v uls --analyze --optimize @call summary-statistics.bat :mysql -vvv qryAnomalyUHFPairs.log :ANALYZEDIVERSITY mysql -vvv qryStatIQVFrequencyBandByCounty.log mysql -vvv qryStatIQVCountyCallsignByBand.log mysql -vvv qryStatIQVFrequencyCallsignByCounty.log :ANALYZEDATAMINING ::testxslt -in tblOrgBigRetail.xml -xsl xsltOrgBigRetail.xsl -out tblOrgBigRetail.arff -validate testxslt -in tblOrgBigRetail.xml -xsl xsltOrgBigRetail.xsl -out tblOrgBigRetail.arff :pause java -Xmx1024m weka.classifiers.trees.J48 -t tblOrgBigRetail.arff >tblOrgBigRetail-weka.txt :ANALYZESOCIALNETWORK :mysql -vvv qryGeoStateCA-dbase.log :del liass-call_sign.png :del liass-call_sign.pdf :del liass-call_sign.svg :@call sna-netdraw.bat tblGeoCountiesSoCal-Frequency :%RPATH% --no-save sna.log @call sna.bat :EXPORTMYSQL @call export-mysql.bat tblDistLocationStateCounty @call export-mysql.bat tblDistLocationStateCountyCity @call export-mysql.bat tblGeoUS @call export-mysql.bat tblGeoStateCA @call export-mysql.bat tblGeoCountiesSoCal @call export-mysql.bat tblGeoCountyLosAngeles @call export-mysql.bat tblGeoCityLosAngeles :@call export-mysql.bat tblGeoCountiesMetroScan :@call export-mysql.bat tblGeoCountiesPoliceCall :@call export-mysql.bat tblGeoCountiesSCAN :@call export-mysql.bat tblGeoCountiesScanFan @call export-mysql.bat tblOrgStateCA @call export-mysql.bat tblOrgCountyLosAngeles @call export-mysql.bat tblOrgCountyLosAngelesMTA @call export-mysql.bat tblOrgCityLosAngeles @call export-mysql.bat tblOrgCityLosAngelesWP @call export-mysql.bat tblOrgAnchor @call export-mysql.bat tblOrgBigRetail @call export-mysql.bat tblOrgLargeSMR @call export-mysql.bat tblOrgCPRA @call export-mysql.bat tblSampleLASDSimplex @call export-mysql.bat tblSampleFederal @call export-mysql.bat tblSampleRailroad :@call export-mysql.bat tblEmulatePoliceCall @call export-mysql.bat tblStatIQVFrequencyBandByCounty @call export-mysql.bat tblStatIQVCountyCallsignByBand @call export-mysql.bat tblStatIQVFrequencyCallsignByCounty :EXPORTSQLITE :@call export-sqlite.bat tblGeoUS @call export-sqlite.bat tblGeoStateCA @call export-sqlite.bat tblGeoCountiesSoCal :EXPORTMSACCESS @call export-msaccess.bat tblGeoStateCA @call export-msaccess.bat tblGeoCountiesSoCal :EXPORTDBASE :mysql -vvv qryGeoStateCA-dbase.log @call export-dbase.bat tblGeoStateCA :mysql -vvv qryCountiesSoCal-dbase.log @call export-dbase.bat tblGeoCountiesSoCal :@call export-ooobase.bat tblGeoStateCA :EXPORTMSEXCEL @call export-spreadsheet.bat tblOrgStateCA copy /y export-spreadsheet.pl export-spreadsheet-pl.txt :(needs work) @call export-spreadsheet-org.bat tblSoCalOrgs copy /y export-spreadsheet-org.pl export-spreadsheet-org-pl.txt :EXPORTFIXEDWIDTH del tblOrgCityLosAngeles-fixedwidth.txt ::del tblStatIQVFrequencyByCounty-fixedwidth.txt ::del tblStatIQVCountyCallsignByBand-fixedwidth.txt ::del tblStatIQVFrequencyCallsignByCounty-fixedwidth.txt mysql -vvv qryExportFixedWidth.log :GENERATESCHEMAS java -jar schemaSpy_2.1.2.jar -cp mysql-connector-java-2.1.12-bin.jar -t mysql -o library -host localhost -db uls -u root :DERIVEFILESIZES perl -w get-filesize.pl tblGeoStateCA-2003.mdb perl -w get-filesize.pl tblGeoStateCA-2000.mdb perl -w get-filesize.pl tblGeoCountiesSoCal-2003.mdb perl -w get-filesize.pl tblGeoCountiesSoCal-2000.mdb move dbase\STATECA5.DBF perl -w get-filesize.pl stateca5.dbf move STATECA5.DBF dbase move dbase\STATECA4.DBF perl -w get-filesize.pl stateca4.dbf move STATECA4.DBF dbase move dbase\STATECA3.DBF perl -w get-filesize.pl stateca3.dbf move STATECA3.DBF dbase move dbase\COSOCAL5.DBF perl -w get-filesize.pl cosocal5.dbf move COSOCAL5.DBF dbase move dbase\COSOCAL4.DBF perl -w get-filesize.pl cosocal4.dbf move COSOCAL4.DBF dbase move dbase\COSOCAL3.DBF perl -w get-filesize.pl cosocal3.dbf move COSOCAL3.DBF dbase perl -w get-filesize.pl tblGeoStateCA.sqlite3x perl -w get-filesize.pl tblGeoStateCA-with-indexes.sqlite3x perl -w get-filesize.pl tblGeoStateCA.sqlite2x perl -w get-filesize.pl tblGeoStateCA-with-indexes.sqlite2x perl -w get-filesize.pl tblGeoCountiesSoCal.sqlite3x perl -w get-filesize.pl tblGeoCountiesSoCal-with-indexes.sqlite3x perl -w get-filesize.pl tblGeoCountiesSoCal.sqlite2x perl -w get-filesize.pl tblGeoCountiesSoCal-with-indexes.sqlite2x perl -w get-filesize.pl tblGeoStateCA.xml perl -w get-filesize.pl tblGeoCountiesSoCal.xml :UPLOAD @call upload.bat mysqlcheck -v uls --analyze --optimize :END net stop MySQL del e:\mysql\.log net start MySQL SET RPATH=