# Congress.gov processing

```elixir
Mix.install([
  {:saxy, "~> 1.6"},
  {:kino, "~> 0.19.0"},
  {:kino_db, "~> 0.4.0"},
  {:adbc, "~> 0.8"},
])
```

## Section

## Bill.dtd parser (congressional bills)

```elixir
{:ok, legis} = File.read("/home/calliope/share/gram/legal/legis.xml")
```

```elixir
{:ok, response} = Saxy.SimpleForm.parse_string(legis)
# response |> Ymlr.Encode.to_s(legis) |> IO.puts
```

```elixir
# defmodule Legis do
#   @behaviour Saxy.Handler

#   def handle_event(:start_document, prolog, state) do
#     # IO.inspect("Start parsing document")
#     {:ok, state}
#   end

#   def handle_event(:end_document, _data, state) do
#     # IO.inspect("Finish parsing document")
#     {:ok, state}
#   end

#   def handle_event(:start_element, {"external-xref", attributes}, state) do
#     # IO.inspect("Start parsing element #{name} with attributes #{inspect(attributes)}")
#     {:ok, [{:ref, attributes} | state]}
#   end
#   def handle_event(:start_element, {name, attributes}, state) do
#     IO.inspect("Start parsing element #{name} with attributes #{inspect(attributes)}")
#     {:ok, state}
#   end

#   def handle_event(:end_element, name, state) do
#     IO.inspect("Finish parsing element #{name}")
#     {:ok, state}
#   end

#   def handle_event(:characters, chars, state) do
#     IO.inspect("Receive characters #{chars}")
#     {:ok, state}
#   end

#   def handle_event(:cdata, cdata, state) do
#     IO.inspect("Receive CData #{cdata}")
#     {:ok, state}
#   end
# end

# def handle_event(:start_element, {name, attributes}, state), do: {:ok, state}
# def handle_event(:start_document, prolog, state), do: {:ok, state}
# def handle_event(:end_document, _data, state),    do: {:ok, state}
# def handle_event(:end_element, name, state),      do: {:ok, state}
# def handle_event(:characters, chars, state),      do: {:ok, state}
# def handle_event(:cdata, cdata, state),           do: {:ok, state}
```

```elixir
defmodule UsCongress.BillDtd.Reference do
  @behaviour Saxy.Handler

  def handle_event(:start_element, {"external-xref", body}, parsed), do:
    {:ok, [(body |> Map.new)["parsable-cite"] | parsed]}

  def handle_event(:start_element, {"dc:title", attributes}, parsed), do:
    {:ok, [:name | parsed]}
  def handle_event(:characters, chars, [:name| parsed]), do:
    {:ok, [{:name, chars} | parsed]}

  def handle_event(:start_element, {"legis-num", attributes}, parsed), do:
    {:ok, [:num | parsed]}
  def handle_event(:characters, chars, [:num| parsed]), do:
    {:ok, [{:num, chars} | parsed]}

  def handle_event(_, _, parsed), do: {:ok, parsed}
end
```

```elixir
Saxy.parse_string(legis, UsCongress.BillDtd.Reference, [])
```

```elixir
Adbc.download_driver!(:duckdb)
db = Kino.start_child!({Adbc.Database, driver: :duckdb})
conn = Kino.start_child!({Adbc.Connection, database: db})
```

```elixir

congresses = (
  (113..119)
  |> Enum.map(& "/chip/cache/hyperdemocracy/usc-textversion/data/usc-#{&1}-textversion.parquet")
  |> Enum.map(& Adbc.Connection.query!(conn, "SELECT tv_xml FROM read_parquet($1)", [&1]))
)

congresses |> hd

# congress = congri |> hd()
# Adbc.Connection.query!(conn, "SELECT tv_xml FROM read_parquet($1)", [congress])

# path = Kino.FS.file_path("usc-119-textversion.parquet")
```

```elixir
legislation = (
  congresses
  |> Enum.flat_map(fn congress ->
    congress.data
    |> Enum.map(& (&1 |> hd |> Adbc.Column.to_list))
  end)
  |> List.flatten()
)
```

```elixir
# legacy sequential logic, benchmarked at:
# 92.7s before parallelization
# see "Generic Processor" section for upgrade.

# legislation
# |> Enum.map(fn legis ->
#   {:ok, parsed} = Saxy.parse_string(legis, UsCongress.BillDtd.Reference, [])
#   parsed
# end)
```

## US Code parser

```elixir
defmodule UsCongress.USCode.Reference do
  @behaviour Saxy.Handler

  def handle_event(:start_element, {"ref", body}, parsed) do
    map = body |> Map.new
    {:ok, [(map["href"] || map) | parsed]}
  end

  def handle_event(:start_element, {"dc:title", _}, parsed), do:
    {:ok, [:name | parsed]}
  def handle_event(:characters, chars, [:name| parsed]), do:
    {:ok, [{:name, chars} | parsed]}


  def handle_event(_, _, parsed), do: {:ok, parsed}
end
```

```elixir
base = "/chip/cache/us-congress.usc.119-102.release"

sources = "#{base}/xml/usc*.xml" |> Path.wildcard()

# # source: ($base)/xml/($label).xml
# # produces: ($base)/toc/($label).toc.yml

sources
|> Enum.map(fn source -> Task.async(fn ->
  {:ok, refs} = (
    source
  |> File.read!()
  |> Saxy.parse_string(UsCongress.USCode.Reference, [])
  )
  refs |> Enum.reverse
end) end)
|> Task.await_many()
```

## Generic Processing

```elixir
defmodule Processor.Source do
  # [upgrade] use Saxy.parse_stream, and file i/o streams.
  def read(glob, parser) do
    glob
    |> Path.wildcard()
    |> Enum.map(&File.read!/1)
    |> map_xml(parser)
  end

  # [upgrade] use Saxy.parse_stream, and file i/o streams.
  # sources: an enumerable of xml strings
  # parser: a Saxy.XML parser
  def map_xml(sources, parser) do
    sources
    |> Enum.map(fn source -> Task.async(fn ->
      {:ok, refs} = source |> Saxy.parse_string(parser, [])
      refs |> Enum.reverse
    end) end)
    |> Task.await_many()
  end
end

```

```elixir
base = "/chip/cache/us-congress.usc.119-102.release"
glob = "#{base}/xml/usc*.xml"
Processor.Source.read(glob, UsCongress.USCode.Reference)
```

```elixir
parsed_bills =
  legislation |> Processor.Source.parse(UsCongress.BillDtd.Reference)
```