Skip to content

Commit

Permalink
First working version (read) of GeoArrow..
Browse files Browse the repository at this point in the history
  • Loading branch information
evetion committed Sep 24, 2024
1 parent 14aa765 commit 1856a42
Show file tree
Hide file tree
Showing 11 changed files with 256 additions and 9 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/CI.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ jobs:
matrix:
version:
- '1.6'
- '1.9'
- '1'
- 'nightly'
os:
- ubuntu-latest
Expand Down
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,6 @@
*.jl.mem
/Manifest.toml
/docs/build/
test/.cpenv
.CondaPkg
*.arrow
3 changes: 3 additions & 0 deletions CondaPkg.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
[deps]
pyarrow = ""
geoarrow-pyarrow = ""
18 changes: 17 additions & 1 deletion Project.toml
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,27 @@ uuid = "5bc3a8d9-1bfb-4624-ba94-a391279174d6"
authors = ["Maarten Pronk <[email protected]> and contributors"]
version = "0.1.0"

[deps]
Arrow = "69666777-d1a9-59fb-9406-91d4454c9d45"
Extents = "411431e0-e8b7-467b-b5e0-f676ba4f2910"
GeoFormatTypes = "68eda718-8dee-11e9-39e7-89f7f65f511f"
GeoInterface = "cf35fbd7-0cd7-5166-be24-54bfbe79505f"
JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1"
WellKnownGeometry = "0f680547-7be7-4555-8820-bb198eeb646b"

[compat]
Arrow = "2.4"
GeoFormatTypes = "0.4"
GeoInterface = "1.2"
JSON3 = "1.14"
WellKnownGeometry = "0.2"
julia = "1.6"

[extras]
CondaPkg = "992eb4ea-22a4-4c89-a5bb-47a3300528ab"
Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
PythonCall = "6099a3de-0909-46bc-b1f4-468b9a2dfc0d"
Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

[targets]
test = ["Test"]
test = ["Test", "Downloads", "CondaPkg", "PythonCall"]
11 changes: 7 additions & 4 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
# GeoArrow
A [geoarrow]() implementation in Julia.

[![Stable](https://img.shields.io/badge/docs-stable-blue.svg)](https://evetion.github.io/GeoArrow.jl/stable/)
[![Dev](https://img.shields.io/badge/docs-dev-blue.svg)](https://evetion.github.io/GeoArrow.jl/dev/)
[![Build Status](https://github.com/evetion/GeoArrow.jl/actions/workflows/CI.yml/badge.svg?branch=main)](https://github.com/evetion/GeoArrow.jl/actions/workflows/CI.yml?query=branch%3Amain)
[![Coverage](https://codecov.io/gh/evetion/GeoArrow.jl/branch/main/graph/badge.svg)](https://codecov.io/gh/evetion/GeoArrow.jl)
*work in progress*

[![Stable](https://img.shields.io/badge/docs-stable-blue.svg)](https://juliageo.github.io/GeoArrow.jl/stable/)
[![Dev](https://img.shields.io/badge/docs-dev-blue.svg)](https://juliageo.github.io/GeoArrow.jl/dev/)
[![Build Status](https://github.com/juliageo/GeoArrow.jl/actions/workflows/CI.yml/badge.svg?branch=main)](https://github.com/juliageo/GeoArrow.jl/actions/workflows/CI.yml?query=branch%3Amain)
[![Coverage](https://codecov.io/gh/juliageo/GeoArrow.jl/branch/main/graph/badge.svg)](https://codecov.io/gh/juliageo/GeoArrow.jl)
[![PkgEval](https://JuliaCI.github.io/NanosoldierReports/pkgeval_badges/G/GeoArrow.svg)](https://JuliaCI.github.io/NanosoldierReports/pkgeval_badges/report.html)
14 changes: 12 additions & 2 deletions src/GeoArrow.jl
Original file line number Diff line number Diff line change
@@ -1,5 +1,15 @@
module GeoArrow
using Arrow
using GeoInterface
using GeoFormatTypes
using JSON3
using WellKnownGeometry
using Extents

# Write your package code here.
include("type.jl")
include("arrow.jl")
include("io.jl")

end
export read, write

end # module
76 changes: 76 additions & 0 deletions src/arrow.jl
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
ArrowTypes.isstringtype(::ArrowTypes.StructKind) = false

POINT = Symbol("geoarrow.point")
LINESTRING = Symbol("geoarrow.linestring")
POLYGON = Symbol("geoarrow.polygon")
MULTIPOINT = Symbol("geoarrow.multipoint")
MULTILINESTRING = Symbol("geoarrow.multilinestring")
MULTIPOLYGON = Symbol("geoarrow.multipolygon")
WKB = Symbol("geoarrow.wkb")
WKT = Symbol("geoarrow.wkt")
BOX = Symbol("geoarrow.box")

function ArrowTypes.JuliaType(::Val{POINT}, x, metadata)
D = length(x.types)
T = x.types[1]
return Geometry{PointTrait,D,T}
end
ArrowTypes.JuliaType(::Val{LINESTRING}, x, metadata) = Geometry{LineStringTrait}
ArrowTypes.JuliaType(::Val{POLYGON}, x, metadata) = Geometry{PolygonTrait}
ArrowTypes.JuliaType(::Val{MULTIPOINT}, x, metadata) = Geometry{MultiPointTrait}
ArrowTypes.JuliaType(::Val{MULTILINESTRING}, x, metadata) = Geometry{MultiLineStringTrait}
ArrowTypes.JuliaType(::Val{MULTIPOLYGON}, x, metadata) = Geometry{MultiPolygonTrait}
ArrowTypes.JuliaType(::Val{WKB}, x, metadata) = GeoFormatTypes.WellKnownBinary
ArrowTypes.JuliaType(::Val{WKT}, x, metadata) = GeoFormatTypes.WellKnownText
function ArrowTypes.JuliaType(::Val{BOX}, x, metadata)
D = length(x.types)
if D == 4
Extents.Extent{(:X, :Y)}
elseif D == 6
Extents.Extent{(:X, :Y, :Z)}
elseif D == 8
Extents.Extent{(:X, :Y, :Z, :M)}
else
throw(ArgumentError("Invalid number of dimensions for Extent"))
end
end
ArrowTypes.ArrowKind(::Type{Geometry}) = ArrowTypes.ListKind()
ArrowTypes.ArrowKind(::Type{<:Geometry{PointTrait,D,T}}) where {D,T} = ArrowTypes.FixedSizeListKind{D,T}()
ArrowTypes.ArrowKind(::Type{<:GeoFormatTypes.WellKnownBinary}) = ArrowTypes.PrimitiveKind()
ArrowTypes.ArrowKind(::Type{<:GeoFormatTypes.WellKnownText}) = ArrowTypes.ListKind()

ArrowTypes.ArrowType(::Type{<:GeoFormatTypes.WellKnownBinary}) = Vector{UInt8}
ArrowTypes.ArrowType(::Type{<:GeoFormatTypes.WellKnownText}) = String
ArrowTypes.ArrowType(::Type{Geometry{X,D,T,G}}) where {X,D,T,G} = G

ArrowTypes.arrowname(::Type{Geometry{PointTrait}}) = POINT
ArrowTypes.arrowname(::Type{Geometry{LineStringTrait}}) = LINESTRING
ArrowTypes.arrowname(::Type{Geometry{PolygonTrait}}) = POLYGON
ArrowTypes.arrowname(::Type{Geometry{MultiPointTrait}}) = MULTIPOINT
ArrowTypes.arrowname(::Type{Geometry{MultiLineStringTrait}}) = MULTILINESTRING
ArrowTypes.arrowname(::Type{Geometry{MultiPolygonTrait}}) = MULTIPOLYGON
ArrowTypes.arrowname(::Type{<:GeoFormatTypes.WellKnownBinary}) = WKB
ArrowTypes.arrowname(::Type{<:GeoFormatTypes.WellKnownText}) = WKT
ArrowTypes.arrowname(::Type{Extents.Extent}) = BOX

ArrowTypes.toarrow(x::Geometry) = x.geom
ArrowTypes.toarrow(x::GeoFormatTypes.WellKnownText) = GeoFormatTypes.val(x)
ArrowTypes.toarrow(x::GeoFormatTypes.WellKnownBinary) = GeoFormatTypes.val(x)
ArrowTypes.toarrow(x::Extents.Extent{(:X, :Y)}) = (; xmin=ex.X[1], ymin=ex.Y[1], xmax=ex.X[2], ymax=ex.Y[2])
ArrowTypes.toarrow(x::Extents.Extent{(:X, :Y, :Z)}) = (; xmin=ex.X[1], ymin=ex.Y[1], zmin=ex.Z[1], xmax=ex.X[2], ymax=ex.Y[2], zmax=ex.Z[2])
ArrowTypes.toarrow(x::Extents.Extent{(:X, :Y, :Z, :M)}) = (; xmin=ex.X[1], ymin=ex.Y[1], zmin=ex.Z[1], mmin=ex.M[1], xmax=ex.X[2], ymax=ex.Y[2], zmax=ex.Z[2], mmax=ex.M[2])

ArrowTypes.fromarrow(::Type{GeoFormatTypes.WellKnownBinary}, x) = GeoFormatTypes.WellKnownBinary(GeoFormatTypes.Geom(), x)
ArrowTypes.fromarrow(::Type{GeoFormatTypes.WellKnownText}, x) = GeoFormatTypes.WellKnownText(GeoFormatTypes.Geom(), x)

function ArrowTypes.fromarrow(::Type{Geometry{X}}, x) where {X}
nt = nested_eltype(x)
D = length(nt.types)
T = nt.types[1]
return Geometry{X,D,T}(x)
end
ArrowTypes.fromarrow(::Type{Extents.Extent}, x) = Extents.Extent(X=(x.xmin, x.xmax), Y=(x.ymin, x.ymax))

nested_eltype(x) = nested_eltype(typeof(x))
nested_eltype(::Type{T}) where {T<:AbstractArray} = nested_eltype(eltype(T))
nested_eltype(::Type{T}) where {T} = T
25 changes: 25 additions & 0 deletions src/io.jl
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
"""
write(path, table; kwargs...)
Write a geospatial table to a file. Like Arrow.write, but with geospatial metadata.
Any kwargs are passed to Arrow.write.
"""
function write(path, t; kwargs...)
projjson = ""
crs = Dict("crs" => projjson)
colmetadata =
Dict(:geometry => ["ARROW:extension:metadata" => JSON3.write(crs)])
Arrow.write(path, t; colmetadata, kwargs...)
end

"""
read(path; kwargs...)
Read a geospatial table from a file. Like Arrow.Table, but with geospatial metadata.
Any kwargs are passed to Arrow.Table.
"""
function read(path; kwargs...)
t = Arrow.Table(path; kwargs...)
meta = Arrow.getmetadata(t)
return t
end
25 changes: 25 additions & 0 deletions src/type.jl
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
struct Geometry{X,D,T,G}
geom::G
end
Base.show(io::IO, x::Geometry{X,D,T}) where {X,D,T} = print(io, "$X geometry in $(D)D with eltype $T")
Geometry{X,D,T}(x) where {X,D,T} = Geometry{X,D,T,typeof(x)}(x)
Geometry{PointTrait}(x::Vararg{T,D}) where {T,D} = Geometry{PointTrait,D,T}(reinterpret(NTuple{D,T}, x))
Geometry{PointTrait,D,T}(x, y, z) where {T,D} = Geometry{PointTrait,D,T}((x, y, z))
Geometry{PointTrait,D,T}(x, y) where {T,D} = Geometry{PointTrait,D,T}((x, y))

Base.getindex(x::Geometry{X,D,T}, i) where {X,D,T} = Geometry{childtrait(X()),D,T}(Base.getindex(x.geom, i))
Base.getindex(x::Geometry{PointTrait,D,T}, i) where {D,T} = Base.getindex(x.geom, i)

GeoInterface.isgeometry(::Type{<:Geometry}) = true
GeoInterface.ncoord(_, ::Geometry{X,D}) where {X,D} = D
GeoInterface.getcoord(::PointTrait, g::Geometry, i) = Base.getindex(g.geom, i)
GeoInterface.geomtrait(::Geometry{X}) where {X} = X()
GeoInterface.ngeom(_, g::Geometry) = length(g.geom)
GeoInterface.getgeom(_, g::Geometry, i) = Base.getindex(g, i)

childtrait(::LineStringTrait) = PointTrait
childtrait(::LinearRingTrait) = PointTrait
childtrait(::PolygonTrait) = LinearRingTrait
childtrait(::MultiPointTrait) = PointTrait
childtrait(::MultiLineStringTrait) = LineStringTrait
childtrait(::MultiPolygonTrait) = PolygonTrait
36 changes: 36 additions & 0 deletions test/links.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
https://raw.githubusercontent.com/geopandas/geopandas/refs/heads/main/geopandas/io/tests/data/geoarrow/example-point.arrow
https://raw.githubusercontent.com/geopandas/geopandas/refs/heads/main/geopandas/io/tests/data/geoarrow/example-point-interleaved.arrow
https://raw.githubusercontent.com/geopandas/geopandas/refs/heads/main/geopandas/io/tests/data/geoarrow/example-point-wkb.arrow
https://raw.githubusercontent.com/geopandas/geopandas/refs/heads/main/geopandas/io/tests/data/geoarrow/example-linestring.arrow
https://raw.githubusercontent.com/geopandas/geopandas/refs/heads/main/geopandas/io/tests/data/geoarrow/example-linestring-interleaved.arrow
https://raw.githubusercontent.com/geopandas/geopandas/refs/heads/main/geopandas/io/tests/data/geoarrow/example-linestring-wkb.arrow
https://raw.githubusercontent.com/geopandas/geopandas/refs/heads/main/geopandas/io/tests/data/geoarrow/example-polygon.arrow
https://raw.githubusercontent.com/geopandas/geopandas/refs/heads/main/geopandas/io/tests/data/geoarrow/example-polygon-interleaved.arrow
https://raw.githubusercontent.com/geopandas/geopandas/refs/heads/main/geopandas/io/tests/data/geoarrow/example-polygon-wkb.arrow
https://raw.githubusercontent.com/geopandas/geopandas/refs/heads/main/geopandas/io/tests/data/geoarrow/example-multipoint.arrow
https://raw.githubusercontent.com/geopandas/geopandas/refs/heads/main/geopandas/io/tests/data/geoarrow/example-multipoint-interleaved.arrow
https://raw.githubusercontent.com/geopandas/geopandas/refs/heads/main/geopandas/io/tests/data/geoarrow/example-multipoint-wkb.arrow
https://raw.githubusercontent.com/geopandas/geopandas/refs/heads/main/geopandas/io/tests/data/geoarrow/example-multilinestring.arrow
https://raw.githubusercontent.com/geopandas/geopandas/refs/heads/main/geopandas/io/tests/data/geoarrow/example-multilinestring-interleaved.arrow
https://raw.githubusercontent.com/geopandas/geopandas/refs/heads/main/geopandas/io/tests/data/geoarrow/example-multilinestring-wkb.arrow
https://raw.githubusercontent.com/geopandas/geopandas/refs/heads/main/geopandas/io/tests/data/geoarrow/example-multipolygon.arrow
https://raw.githubusercontent.com/geopandas/geopandas/refs/heads/main/geopandas/io/tests/data/geoarrow/example-multipolygon-interleaved.arrow
https://raw.githubusercontent.com/geopandas/geopandas/refs/heads/main/geopandas/io/tests/data/geoarrow/example-multipolygon-wkb.arrow
https://raw.githubusercontent.com/geopandas/geopandas/refs/heads/main/geopandas/io/tests/data/geoarrow/example-point_z.arrow
https://raw.githubusercontent.com/geopandas/geopandas/refs/heads/main/geopandas/io/tests/data/geoarrow/example-point_z-interleaved.arrow
https://raw.githubusercontent.com/geopandas/geopandas/refs/heads/main/geopandas/io/tests/data/geoarrow/example-point_z-wkb.arrow
https://raw.githubusercontent.com/geopandas/geopandas/refs/heads/main/geopandas/io/tests/data/geoarrow/example-linestring_z.arrow
https://raw.githubusercontent.com/geopandas/geopandas/refs/heads/main/geopandas/io/tests/data/geoarrow/example-linestring_z-interleaved.arrow
https://raw.githubusercontent.com/geopandas/geopandas/refs/heads/main/geopandas/io/tests/data/geoarrow/example-linestring_z-wkb.arrow
https://raw.githubusercontent.com/geopandas/geopandas/refs/heads/main/geopandas/io/tests/data/geoarrow/example-polygon_z.arrow
https://raw.githubusercontent.com/geopandas/geopandas/refs/heads/main/geopandas/io/tests/data/geoarrow/example-polygon_z-interleaved.arrow
https://raw.githubusercontent.com/geopandas/geopandas/refs/heads/main/geopandas/io/tests/data/geoarrow/example-polygon_z-wkb.arrow
https://raw.githubusercontent.com/geopandas/geopandas/refs/heads/main/geopandas/io/tests/data/geoarrow/example-multipoint_z.arrow
https://raw.githubusercontent.com/geopandas/geopandas/refs/heads/main/geopandas/io/tests/data/geoarrow/example-multipoint_z-interleaved.arrow
https://raw.githubusercontent.com/geopandas/geopandas/refs/heads/main/geopandas/io/tests/data/geoarrow/example-multipoint_z-wkb.arrow
https://raw.githubusercontent.com/geopandas/geopandas/refs/heads/main/geopandas/io/tests/data/geoarrow/example-multilinestring_z.arrow
https://raw.githubusercontent.com/geopandas/geopandas/refs/heads/main/geopandas/io/tests/data/geoarrow/example-multilinestring_z-interleaved.arrow
https://raw.githubusercontent.com/geopandas/geopandas/refs/heads/main/geopandas/io/tests/data/geoarrow/example-multilinestring_z-wkb.arrow
https://raw.githubusercontent.com/geopandas/geopandas/refs/heads/main/geopandas/io/tests/data/geoarrow/example-multipolygon_z.arrow
https://raw.githubusercontent.com/geopandas/geopandas/refs/heads/main/geopandas/io/tests/data/geoarrow/example-multipolygon_z-interleaved.arrow
https://raw.githubusercontent.com/geopandas/geopandas/refs/heads/main/geopandas/io/tests/data/geoarrow/example-multipolygon_z-wkb.arrow
52 changes: 51 additions & 1 deletion test/runtests.jl
Original file line number Diff line number Diff line change
@@ -1,6 +1,56 @@
using GeoArrow
using Arrow
using GeoInterface
using Downloads
using Test
# ENV["JULIA_CONDAPKG_OFFLINE"] = true
ENV["JULIA_CONDAPKG_ENV"] = joinpath(@__DIR__, ".cpenv")
using PythonCall
# ga = pyimport("geoarrow.pyarrow")
feather = pyimport("pyarrow.feather")

@testset "GeoArrow.jl" begin
# Write your tests here.
@testset "Test datasets" begin
# Data taken from the geopandas tests, courtesy of Joris Van den Bossche
for url in readlines("links.txt")
fn = joinpath("data", split(url, "/")[end])
isfile(fn) && continue
try
@info "Downloading $fn"
Downloads.download(url, fn)
catch
@warn "Failed to download $fn"
end
end

for arrowfn in filter(endswith("arrow"), readdir("data", join=true))
@testset "$arrowfn" begin
t = Arrow.Table(arrowfn)
geom = t.geometry[1]
@test GeoInterface.isgeometry(geom)
@test GeoInterface.geomtrait(geom) isa GeoInterface.AbstractGeometryTrait
@test GeoInterface.ncoord(geom) in [2, 3]
@test GeoInterface.testgeometry(geom)

io = IOBuffer()
GeoArrow.write(io, t; compress=:zstd)
seekstart(io)
nt = GeoArrow.read(io, convert=true)
ngeom = t.geometry[1]
@test GeoInterface.isgeometry(geom)

@test ngeom == geom
end
end
end
@testset "Python" begin
for arrowfn in filter(endswith("arrow"), readdir("data", join=true))
@testset "$arrowfn" begin
t = Arrow.Table(arrowfn)
fn = joinpath("data/write", basename(arrowfn))
GeoArrow.write(fn, t)
pt = feather.read_table(fn)
end
end
end
end

0 comments on commit 1856a42

Please sign in to comment.