[WIP] Mutable memory mapping

src/LasIO.jl

@@ -45,6 +45,9 @@ export
     raw_classification,
     flag_byte,
 
+    # Functions on PointVector
+    sync 
+
     # extended from ColorTypes
     red,
     green,

src/fileio.jl

@@ -17,13 +17,17 @@ end
 # skip the LAS file's magic four bytes, "LASF"
 skiplasf(s::Union{Stream{format"LAS"}, Stream{format"LAZ"}, IO}) = read(s, UInt32)
 
-function load(f::File{format"LAS"}; mmap=false)
-    open(f) do s
-        load(s; mmap=mmap)
+function load(f::File{format"LAS"}; mmap=false, mutable=false)
+    # file, readable, writable, create, truncate, append
+    open(f, true, mutable, false, false, false) do s
+        load(s; mmap=mmap, mutable=mutable)
     end
 end
 
-# Load pipe separately since it can't be memory mapped
+# mutable implies memorymapping
+load!(f::File{format"LAS"}) = load(f; stream=true, mutable=true)
+
+# Load pipe apart since it can't be memory mapped
 function load(s::Pipe)
     skiplasf(s)
     header = read(s, LasHeader)
@@ -37,7 +41,7 @@ function load(s::Pipe)
     header, pointdata
 end
 
-function load(s::Stream{format"LAS"}; mmap=false)
+function load(s::Stream{format"LAS"}; mmap=false, mutable=false)
     skiplasf(s)
     header = read(s, LasHeader)
 
@@ -47,7 +51,7 @@ function load(s::Stream{format"LAS"}; mmap=false)
     if mmap
         pointsize = Int(header.data_record_length)
         pointbytes = Mmap.mmap(s.io, Vector{UInt8}, n*pointsize, position(s))
-        pointdata = PointVector{pointtype}(pointbytes, pointsize)
+        pointdata = PointVector{pointtype}(pointbytes, pointsize, mutable=mutable)
     else
         pointdata = Vector{pointtype}(n)
         for i=1:n

src/point.jl

@@ -7,34 +7,46 @@ Custom PointVector struct for memory mapped LasPoints.
 Inspiration taken from UnalignedVector.jl
 and extended it with custom indexing and packing.
 """
-struct PointVector{T} <: AbstractArray{T,1}
+struct PointVector{T <: LasPoint} <: AbstractArray{T,1}
+    data::Vector{UInt8}
     io::IOBuffer
     n::Int
     pointsize::Int
 
-    function PointVector{T}(data::Vector{UInt8}, pointsize::Integer) where {T}
+    function PointVector{T}(data::Vector{UInt8}, pointsize::Integer; mutable=false) where T <: LasPoint
         n = length(data) ÷ pointsize
 
         # IOBuffer takes (data, readable, writable)
-        new{T}(IOBuffer(data, true, false), n, pointsize)
+        new{T}(data, IOBuffer(data, true, mutable), n, pointsize)
     end
 end
 
 Base.IndexStyle(::Type{PointVector{T}}) where {T} = IndexLinear()
 Base.length(pv::PointVector) = pv.n
 Base.size(pv::PointVector) = (length(pv),)
 
-function Base.getindex(pv::PointVector{T}, i::Int) where {T}
+function sync(pv::PointVector{T}) where T <: LasPoint
+    flush(pv.io)
+    Mmap.sync!(pv.data)
+end
+
+function Base.close(pv::PointVector{T}) where T <: LasPoint
+    close(pv.io)
+    close(pv.data)
+    # delete pv?
+end
+
+function Base.getindex(pv::PointVector{T}, i::Int) where T <: LasPoint
     offset = (i-1) * pv.pointsize  # seeking uses 0 based indexing!
     position(pv.io) != offset && seek(pv.io, offset)
     read(pv.io, T)
 end
 
-function Base.setindex!(pv::PointVector{T}, val::T, i::Int) where {T}
-    # offset = (i-1) * pv.pointsize  # seeking uses 0 based indexing!
-    # position(pv.io) != offset && seek(pv.io, offset)
-    # write(pv.io, val)
-    error("Can't write to read only memory mapped file.")
+function Base.setindex!(pv::PointVector{T}, val::T, i::Int) where T <: LasPoint
+    offset = (i-1) * pv.pointsize  # seeking uses 0 based indexing!
+    position(pv.io) != offset && seek(pv.io, offset)
+    write(pv.io, val)
+    flush(pv.io)
 end
 
 function Base.show(io::IO, pointdata::AbstractVector{<:LasPoint})
@@ -43,7 +55,7 @@ function Base.show(io::IO, pointdata::AbstractVector{<:LasPoint})
 end
 
 "ASPRS LAS point data record format 0"
-@gen_io struct LasPoint0 <: LasPoint
+@gen_io mutable struct LasPoint0 <: LasPoint
     x::Int32
     y::Int32
     z::Int32
@@ -56,7 +68,7 @@ end
 end
 
 "ASPRS LAS point data record format 1"
-@gen_io struct LasPoint1 <: LasPoint
+@gen_io mutable struct LasPoint1 <: LasPoint
     x::Int32
     y::Int32
     z::Int32
@@ -70,7 +82,7 @@ end
 end
 
 "ASPRS LAS point data record format 2"
-@gen_io struct LasPoint2 <: LasPoint
+@gen_io mutable struct LasPoint2 <: LasPoint
     x::Int32
     y::Int32
     z::Int32
@@ -86,7 +98,7 @@ end
 end
 
 "ASPRS LAS point data record format 3"
-@gen_io struct LasPoint3 <: LasPoint
+@gen_io mutable struct LasPoint3 <: LasPoint
     x::Int32
     y::Int32
     z::Int32
@@ -114,10 +126,6 @@ function Base.show(io::IO, p::LasPoint)
     println(io, "LasPoint(x=$x, y=$y, z=$z, classification=$cl)")
 end
 
-# Extend base by enabling reading/writing relevant FixedPointNumbers from IO.
-Base.read(io::IO, ::Type{N0f16}) = reinterpret(N0f16, read(io, UInt16))
-Base.write(io::IO, t::N0f16) = write(io, reinterpret(UInt16, t))
-
 # functions for IO on points
 
 "X coordinate (Float64), apply scale and offset according to the header"

test/benchmark.jl

@@ -93,4 +93,5 @@ println("New function using streaming")
 @btime test_stream()
 
 @show op, np, sp
-@assert op[5] == np[5] == sp[5]
+@show op[5], np[5], sp[5]
+@assert op[5].x == np[5].x == sp[5].x

test/stream.jl

@@ -19,12 +19,56 @@ rm(writefile)
 srsfile = joinpath(workdir, "srs.las")
 srsfile_out = joinpath(workdir, "srs-out.las")
 srsheader, srspoints = load(srsfile, mmap=true)
+for record in srsheader.variable_length_records
+    @test record.reserved === 0xaabb
+    @test record.user_id == "LASF_Projection"
+    @test typeof(record.description) == String
+    if record.record_id == 34735
+        @test record.data.key_directory_version === UInt16(1)
+        @test record.data.key_reversion === UInt16(1)
+        @test record.data.minor_revision === UInt16(0)
+        @test record.data.number_of_keys === UInt16(length((record.data.keys)))
+        @test typeof(record.data.keys) == Vector{LasIO.KeyEntry}
+    end
+end
+@test srsheader.version_major == 1
+@test srsheader.version_minor == 0
+@test srsheader.data_format_id == 1
+@test srsheader.n_vlr == 3
+@test isa(srsheader.variable_length_records, Vector{LasVariableLengthRecord})
+for vlr in srsheader.variable_length_records
+    @test vlr.reserved === 0xaabb
+    @test vlr.user_id == "LASF_Projection"
+    @test vlr.description == ""
+end
+
+@test srsheader.variable_length_records[1].record_id == LasIO.id_geokeydirectorytag
+@test srsheader.variable_length_records[2].record_id == LasIO.id_geodoubleparamstag
+@test srsheader.variable_length_records[3].record_id == LasIO.id_geoasciiparamstag
+@test typeof(srsheader.variable_length_records[1].data) == LasIO.GeoKeys
+@test typeof(srsheader.variable_length_records[2].data) == LasIO.GeoDoubleParamsTag
+@test typeof(srsheader.variable_length_records[3].data) == LasIO.GeoAsciiParamsTag
+
+@test isnull(LasIO.epsg_code(header))
+@test LasIO.epsg_code(srsheader) === Nullable{Int}(32617)
+# set the SRS. Note: this will not change points, but merely set SRS-metadata.
+epsgheader = deepcopy(srsheader)
+LasIO.epsg_code!(epsgheader, 32633)  # set to WGS 84 / UTM zone 33N, not the actual SRS
+@test epsgheader.variable_length_records[1].record_id == LasIO.id_geokeydirectorytag
+@test count(LasIO.is_srs, srsheader.variable_length_records) == 3
+@test count(LasIO.is_srs, epsgheader.variable_length_records) == 1
+
 save(srsfile_out, srsheader, srspoints)
 @test hash(read(srsfile)) == hash(read(srsfile_out))
 rm(srsfile_out)
 
 # Test editing stream file
 srsfile = joinpath(workdir, "srs.las")
-srsfile_out = joinpath(workdir, "srs-out.las")
-srsheader, srspoints = load(srsfile, mmap=true)
-@test_throws ErrorException srspoints[5] = LasPoint1(1,1,1,1,1,1,1,1,1,1.0)
+srsheader, srspoints = load(srsfile, mmap=true, mutable=true)
+p = srspoints[5]
+p.x = 2
+srspoints[5] = p
+sync(srspoints)  # necessary?
+
+_, points = load(srsfile, mmap=false)
+@test points[5].x == 2