Merge pull request #12 from tylerjthomas9/dev

Improve progress bars, rename get_quarterly_filings to download_filings

Author: tylerjthomas9

Project.toml

@@ -1,22 +1,22 @@
 name = "ScrapeSEC"
 uuid = "856806e7-be2f-4540-8165-3a51303b7af0"
 authors = ["tylerjthomas9 <tylerjthomas9@gmail.com>"]
-version = "0.6.2"
+version = "0.7.0"
 
 [deps]
 CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
 DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
-ProgressMeter = "92933f4c-e287-5a05-a399-4b506db050ca"
+Term = "22787eb5-b846-44ae-b979-8e399b8463ab"
 ZipFile = "a5390f91-8eb1-5f08-bee0-b1d1ffed6cea"
 
 [compat]
 CSV = "0.9, 0.10"
 DataFrames = "1"
 HTTP = "1"
-ProgressMeter = "1"
-ZipFile = "0.9"
+Term = "1"
+ZipFile = "0.9, 0.10"
 julia = "1"
 
 [extras]

README.md

@@ -10,44 +10,42 @@
 
 ## Installation
 
+From the Julia REPL:
 ```julia
-julia> using Pkg; Pkg.add("ScrapeSEC")
+julia> ]  # enters the pkg interface
+pkg> add ScrapeSEC
 ```
 
 ```julia
-]add ScrapeSEC
+julia> using Pkg; Pkg.add(ScrapeSEC)
 ```
 
 From source:
 ```julia
-julia> using Pkg; Pkg.add(url="https://github.com/tylerjthomas9/ScrapeSEC.jl/")
+julia> ]  # enters the pkg interface
+pkg> add https://github.com/tylerjthomas9/ScrapeSEC.jl
 ```
-
-```julia
-]add https://github.com/tylerjthomas9/ScrapeSEC.jl/
-```
-
 # Examples
 
-Download filing metadata for 2012-2020 from the [SEC archives](https://www.sec.gov/Archives/).
+Download filing metadata for 2020-2022 from the [SEC archives](https://www.sec.gov/Archives/).
 
 ```julia
-using ScrapeSEC: get_metadata
-download_metadata_files(2012, 2020)
+using ScrapeSEC
+download_metadata_files(2020, 2022)
 ```
 
-Download 10-K, 8-K, and 10-Q metadata, filings for 2012-2020
+Download 10-K, 8-K, and 10-Q metadata, filings for 2020-2022
 
 ```julia
-using ScrapeSEC: get_quarterly_filings
-get_quarterly_filings(2012, 2020; filing_types=["10-K", "8-K", "10-Q"])
+using ScrapeSEC
+download_filings(2020, 2022; filing_types=["10-K", "8-K", "10-Q"])
 ```
 
 
-Download filing metadata for 2012-2020, create a main index file, and download 10-Ks using the master index
+Download filing metadata for 2020-2022, create a main index file, and download 10-Ks using the combined index file
 ```julia
 using ScrapeSEC
-download_metadata_files(2012, 2020)
+download_metadata_files(2020, 2022)
 create_main_index()
-get_quarterly_filings("./metadata/main_idx.tsv"; filing_types=["10-K", ])
+download_filings("./metadata/main_idx.tsv"; filing_types=["10-K", ])
 ```

docs/src/index.md

@@ -3,7 +3,8 @@
 # Installation
 From the Julia REPL:
 ```julia
-julia> ]add ScrapeSEC
+julia> ]  # enters the pkg interface
+pkg> add ScrapeSEC
 ```
 
 ```julia
@@ -12,7 +13,8 @@ julia> using Pkg; Pkg.add(ScrapeSEC)
 
 From source:
 ```julia
-julia> ]add https://github.com/tylerjthomas9/ScrapeSEC.jl
+julia> ]  # enters the pkg interface
+pkg> add https://github.com/tylerjthomas9/ScrapeSEC.jl
 ```
 
 ```julia
@@ -32,6 +34,6 @@ ScrapeSEC.create_main_index
 # Download Filings
 ```@docs
 download_filing
-get_quarterly_filings
+download_filings
 ```
 

src/ScrapeSEC.jl

@@ -6,9 +6,18 @@ using DataFrames
 import Dates
 import CSV
 import HTTP
-using ProgressMeter
+using Term.Progress
 import ZipFile
 
+const progress_bar_columns = [
+    Progress.DescriptionColumn,
+    Progress.SeparatorColumn,
+    Progress.ProgressColumn,
+    Progress.CompletedColumn,
+    Progress.SeparatorColumn,
+    Progress.ETAColumn,
+]
+
 # source files
 include("download_metadata.jl")
 include("main_index.jl")
@@ -20,7 +29,7 @@ export
     create_main_index,
 
     # filing downloaders
-    get_quarterly_filings,
+    download_filings,
     download_filing
 
 end

src/download_filings.jl

@@ -34,30 +34,42 @@ end
 
 """
 ```julia
-function get_quarterly_filings(
+function download_filings(
     metadata_file::String; 
     dest="./data/"::String, 
     filing_types=["10-K", ]::Vector{String}, 
     download_rate=10::Int, 
-    skip_file=true::Bool
+    skip_file=true::Bool,
+    pbar=ProgressBar()::pbar,
+    stop_pbar=true::Bool,
+    pbar_desc="Downloading Filings"::string,
+    running_tests=false::Bool
 )
 ```
 
-Get quarterly filings from https://www.sec.gov/Archives/ using a metadata file
+Download quarterly filings from https://www.sec.gov/Archives/ using a metadata file
 
 Parameters
 * `metadata_file`: CSV file with filing metadata
 * `dest`: Destination folder for downloaded filings
 * `filing_types`: Types of filings to download (eg. ["10-K", "10-Q"])
 * `download_rate`: Number of filings to download every second (limit=10)
 * `skip_file`: If true, existing files will be skipped
+* `pbar`: ProgressBar (Term.jl)
+* `stop_pbar`: If false, progress bar will not be stopped
+* `pbar_desc`: pbar Description
+* `runnings_tests`: If true, only downloads one file
 """
-function get_quarterly_filings(
+function download_filings(
     metadata_file::String;
     dest = "./data/"::String,
     filing_types = ["10-K"]::Vector{String},
     download_rate = 10::Int,
     skip_file = true::Bool,
+    pbar = ProgressBar()::pbar,
+    stop_pbar = true::Bool,
+    pbar_desc = "Downloading Filings"::string,
+    running_tests = false::Bool,
 )
 
     # verify download_rate is valid (less than 10 requests per second, more than 0)
@@ -70,8 +82,6 @@ function get_quarterly_filings(
         )
     end
 
-    println("Metadata: " * metadata_file)
-
     df = CSV.File(metadata_file, delim = "|") |> DataFrame
     df = df[∈(filing_types).(df[!, "Form Type"]), :]
 
@@ -82,8 +92,12 @@ function get_quarterly_filings(
 
     # download filings at 10 requests per second
     sleep_time = 1 / download_rate
-    @showprogress 1 "Downloading Filings..." for row in eachrow(df)
 
+    # setup progress bar
+    job = addjob!(pbar; N = size(df, 1), description = pbar_desc)
+    start!(pbar)
+    for row in eachrow(df)
+        update!(job)
         # check if filing already has been downloaded
         full_file = joinpath(dest, replace(row["Filename"], "edgar/data/" => ""))
         if isfile(full_file) && skip_file
@@ -95,15 +109,22 @@ function get_quarterly_filings(
 
         # rest to throttle api hits to around 10/second
         sleep(sleep_time)
+        render(pbar)
+
+        if running_tests
+            break
+        end
+    end
+    if stop_pbar
+        stop!(pbar)
     end
 
     return
 end
 
-
 """
 ```julia
-function get_quarterly_filings(
+function download_filings(
     start_year::Int, 
     end_year::Int; 
     quarters=[1,2,3,4]::Vector{Int}, 
@@ -112,11 +133,12 @@ function get_quarterly_filings(
     download_rate=10::Int, 
     metadata_dest="./metadata/"::String,
     skip_file=true::Bool, 
-    skip_metadata_file=true::Bool
+    skip_metadata_file=true::Bool,
+    running_tests=false::Bool
 )
 ```
 
-Get quarterly filings from https://www.sec.gov/Archives/
+Download quarterly filings from https://www.sec.gov/Archives/
 
 Parameters
 * `start_year`: First year to download filings
@@ -128,8 +150,9 @@ Parameters
 * `metadata_dest`: Directory to store metadata files
 * `skip_file`: If true, existing files will be skipped
 * `skip_metadata_file`: If true, existing metadata files will be skipped
+* `runnings_tests`: If true, only downloads one file
 """
-function get_quarterly_filings(
+function download_filings(
     start_year::Int,
     end_year::Int;
     quarters = [1, 2, 3, 4]::Vector{Int},
@@ -139,6 +162,7 @@ function get_quarterly_filings(
     metadata_dest = "./metadata/"::String,
     skip_file = true::Bool,
     skip_metadata_file = true::Bool,
+    running_tests = false::Bool,
 )
 
     # get current year, quarter to prevent errors trying to get future data
@@ -163,26 +187,36 @@ function get_quarterly_filings(
         start_year,
         end_year;
         quarters = quarters,
-        #download_rate = download_rate,
         dest = metadata_dest,
         skip_file = skip_metadata_file,
     )
 
 
     # download all quarterly filings
-    file_paths = [
-        joinpath(metadata_dest, string(t[1]) * "-QTR" * string(t[2]) * ".tsv") for
-        t in time_periods
-    ]
-    for file in file_paths
-        get_quarterly_filings(
+    pbar = ProgressBar(columns = progress_bar_columns)
+    job = addjob!(
+        pbar;
+        N = size(time_periods, 1),
+        description = "Iterating Over Time Periods...",
+    )
+    start!(pbar)
+    for t in time_periods
+        update!(job)
+        file = joinpath(metadata_dest, string(t[1]) * "-QTR" * string(t[2]) * ".tsv")
+        download_filings(
             file;
             dest = dest,
             filing_types = filing_types,
             download_rate = download_rate,
             skip_file = skip_file,
+            pbar = pbar,
+            stop_pbar = false,
+            pbar_desc = "Downloading $(t[1]) Q$(t[2]) Filings",
+            running_tests = running_tests,
         )
+        render(pbar)
     end
+    stop!(pbar)
 
     return
 end

src/download_metadata.jl

@@ -69,7 +69,7 @@ function download_metadata(
     end
 
     # download, unzip file
-    HTTP.download(url, temp_zip)
+    HTTP.download(url, temp_zip, update_period = Inf)
     zarchive = ZipFile.Reader(temp_zip)
     for f in zarchive.files
         @assert f.name == "master.idx"
@@ -151,9 +151,21 @@ function download_metadata_files(
     urls = get_metadata_urls(time_periods)
 
     # download metadata files
-    @showprogress 1 "Downloading Metadata..." for idx in eachindex(urls)
-        download_metadata(urls[idx]; dest = dest, skip_file = skip_file, verbose = verbose)
+    n_files = size(urls, 1)
+    pbar = ProgressBar(columns = progress_bar_columns)
+    job = addjob!(pbar; N = n_files, description = "Downloading Metadata CSVs...")
+    start!(pbar)
+    @inbounds for idx in eachindex(urls)
+        update!(job)
+        ScrapeSEC.download_metadata(
+            urls[idx];
+            dest = dest,
+            skip_file = skip_file,
+            verbose = verbose,
+        )
+        render(pbar)
     end
+    stop!(pbar)
 
     return
 end

test/download_filings.jl

@@ -10,19 +10,18 @@
 end
 
 
-@testset "get_quarterly_filings()" begin
+@testset "download_filings()" begin
 
-    get_quarterly_filings(
+    download_filings(
         1994,
         1994;
         quarters = [4],
         dest = "./temp/",
         metadata_dest = "./metadata/",
+        running_tests = true,
     )
     @test isfile("./metadata/1994-QTR4.tsv")
-    @test isfile("./temp/3146/0000950144-94-002172.txt")
     rm("./metadata/1994-QTR4.tsv")
-    # TODO: Is it safe to clear the temp dir? I dont want to accidently delete peoples files who
-    # run the tests
+    # TODO: Is it safe to clear the temp dir? I dont want to accidently user files
 
 end