radolan.go 6.93 KB
Newer Older
Jonny Schäfer's avatar
Jonny Schäfer committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
// Package radolan parses the DWD RADOLAN / RADVOR radar composite format. This data
// is available at the Global Basic Dataset (http://www.dwd.de/DE/leistungen/gds/gds.html).
// The obtained results can be processed and visualized with additional functions.
//
// Currently the national grid [1][4] and the extended european grid [5] are supported.
// Tested input products are PG, FZ, SF, RW, RX and EX. Those can be considered working with
// sufficient accuracy.
//
// In cases, where the publicly available format specification is unprecise or contradictory,
// reverse engineering was used to obtain reasonable approaches.
//	Used references:
//
//	[1] https://www.dwd.de/DE/leistungen/radolan/radolan_info/radolan_radvor_op_komposit_format_pdf.pdf
//	[2] https://www.dwd.de/DE/leistungen/gds/weiterfuehrende_informationen.zip
//	[3]  - legend_radar_products_fz_forecast.pdf
//	[4]  - legend_radar_products_pg_coordinates.pdf
//	[5]  - legend_radar_products_radolan_rw_sf.pdf
//	[6] https://www.dwd.de/DE/leistungen/radarniederschlag/rn_info/download_niederschlagsbestimmung.pdf
package radolan

import (
Jonny Schäfer's avatar
Jonny Schäfer committed
22
	"archive/tar"
Jonny Schäfer's avatar
Jonny Schäfer committed
23
	"bufio"
Jonny Schäfer's avatar
Jonny Schäfer committed
24
	"compress/gzip"
Jonny Schäfer's avatar
Jonny Schäfer committed
25
26
	"fmt"
	"io"
Jonny Schäfer's avatar
Jonny Schäfer committed
27
	"sort"
Jonny Schäfer's avatar
Jonny Schäfer committed
28
29
30
	"time"
)

31
32
33
34
35
36
37
// Radolan radar data is provided as single local sweeps or so called composite
// formats. Each composite is a combined image consisting of mulitiple radar
// sweeps spread over the composite area.
// The 2D composite c has a an internal resolution of c.Dx (horizontal) * c.Dy
// (vertical) records covering a real surface of c.Dx * c.Rx * c.Dy * c.Dy
// square kilometers.
// The pixel value at the position (x, y) is represented by
38
39
40
// c.Data[ y ][ x ] and is stored as raw float value (NaN if the no-data flag
// is set). Some 3D radar products feature multiple layers in which the voxel
// at position (x, y, z) is accessible by c.DataZ[ z ][ y ][ x ].
Jonny Schäfer's avatar
Jonny Schäfer committed
41
//
42
// The data value is used differently depending on the product type:
Jonny Schäfer's avatar
Jonny Schäfer committed
43
// (also consult the DataUnit field of the Composite)
44
//
45
//	Product label            | values represent         | unit
46
47
//	-------------------------+--------------------------+------------------------
//	 PG, PC, PX*, ...        | cloud reflectivity       | dBZ
48
49
//	 RX, WX, EX, FZ, FX, ... | cloud reflectivity	    | dBZ
//	 RW, SF,  ...            | aggregated precipitation | mm/interval
50
51
52
53
//	 PR*, ...                | doppler radial velocity  | m/s
//
// The cloud reflectivity (in dBZ) can be converted to rainfall rate (in mm/h)
// via PrecipitationRate().
Jonny Schäfer's avatar
Jonny Schäfer committed
54
55
56
57
58
//
// The cloud reflectivity factor Z is stored in its logarithmic representation dBZ:
//	dBZ = 10 * log(Z)
// Real world geographical coordinates (latitude, longitude) can be projected into the
// coordinate system of the composite by using the translation method:
Jonny Schäfer's avatar
Jonny Schäfer committed
59
//	// if c.HasProjection
Jonny Schäfer's avatar
Jonny Schäfer committed
60
61
//	x, y := c.Translate(52.51861, 13.40833)	// Berlin (lat, lon)
//
62
63
//	dbz := c.At(int(x), int(y))					// Raw value is Cloud reflectivity (dBZ)
//	rat := radolan.PrecipitationRate(radolan.Doelling98, dbz)	// Rainfall rate (mm/h) using Doelling98 as Z-R relationship
Jonny Schäfer's avatar
Jonny Schäfer committed
64
65
66
67
68
69
//
//	fmt.Println("Rainfall in Berlin [mm/h]:", rat)
//
type Composite struct {
	Product string // composite product label

70
71
72
	CaptureTime  time.Time     // time of source data capture used for forcasting
	ForecastTime time.Time     // data represents conditions predicted for this time
	Interval     time.Duration // time duration until next forecast
Jonny Schäfer's avatar
Jonny Schäfer committed
73

Jonny Schäfer's avatar
Jonny Schäfer committed
74
	DataUnit Unit
75

76
77
78
79
80
81
	PlainData [][]float32 // data for parsed plain data element [y][x]
	Px        int         // plain data width
	Py        int         // plain data height

	DataZ [][][]float32 // data for each voxel [z][y][x] (composites use only one z-layer)
	Data  [][]float32   //  data for each pixel [y][x] at layer 0 (alias for DataZ[0][x][y])
Jonny Schäfer's avatar
Jonny Schäfer committed
82
83
84

	Dx int // data width
	Dy int // data height
85
	Dz int // data layer
Jonny Schäfer's avatar
Jonny Schäfer committed
86
87
88
89

	Rx float64 // horizontal resolution in km/px
	Ry float64 // vertical resolution in km/px

Jonny Schäfer's avatar
Jonny Schäfer committed
90
91
	HasProjection bool // coordinate translation available

Jonny Schäfer's avatar
Jonny Schäfer committed
92
93
	dataLength int // length of binary section in bytes

94
95
	precision int       // multiplicator 10^precision for each raw value
	level     []float32 // maps data value to corresponding index value in runlength based formats
Jonny Schäfer's avatar
Jonny Schäfer committed
96
97
98
99
100

	offx float64 // horizontal projection offset
	offy float64 // vertical projection offset
}

101
102
103
104
105
106
107
108
109
110
// ErrUnknownUnit indicates that the unit of the radar data is not defined in
// the catalog. The data values can be incorrect due to unit dependent
// conversions during parsing. Be careful when further processing the
// composite.
var ErrUnknownUnit = newError("NewComposite", "data unit not defined in catalog. data values can be incorrect")

// NewComposite reads binary data from rd and parses the composite.  An error
// is returned on failure. When ErrUnknownUnit is returned, the data values can
// be incorrect due to unit dependent conversions during parsing. In this case
// be careful when further processing the composite.
Jonny Schäfer's avatar
Jonny Schäfer committed
111
112
113
114
115
116
117
118
119
120
121
122
123
func NewComposite(rd io.Reader) (comp *Composite, err error) {
	reader := bufio.NewReader(rd)
	comp = &Composite{}

	err = comp.parseHeader(reader)
	if err != nil {
		return
	}

	err = comp.parseData(reader)
	if err != nil {
		return
	}
124
	comp.arrangeData()
Jonny Schäfer's avatar
Jonny Schäfer committed
125
126
127

	comp.calibrateProjection()

128
129
130
131
	if comp.DataUnit == Unit_unknown {
		err = ErrUnknownUnit
	}

Jonny Schäfer's avatar
Jonny Schäfer committed
132
133
134
	return
}

Jonny Schäfer's avatar
Jonny Schäfer committed
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
// NewComposites reads tar gz data from rd and returns the parsed composites sorted by
// ForecastTime in ascending order.
func NewComposites(rd io.Reader) ([]*Composite, error) {
	gzipReader, err := gzip.NewReader(rd)
	if err != nil {
		return nil, err
	}
	defer gzipReader.Close()

	tarReader := tar.NewReader(gzipReader)

	var cs []*Composite
	for {
		_, err := tarReader.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return nil, err
		}

		c, err := NewComposite(tarReader)
		if err != nil {
			return nil, err
		}
		cs = append(cs, c)
	}

	// sort composites in chronological order
	sort.Slice(cs, func(i, j int) bool { return cs[i].ForecastTime.Before(cs[j].ForecastTime) })
	return cs, nil
}

Jonny Schäfer's avatar
Jonny Schäfer committed
168
169
170
171
172
173
174
175
// NewDummy creates a blank dummy composite with the given product label and dimensions. It can
// be used for generic coordinate translation.
func NewDummy(product string, dx, dy int) (comp *Composite) {
	comp = &Composite{Product: product, Dx: dx, Dy: dy}
	comp.calibrateProjection()
	return
}

176
177
178
179
// At is shorthand for c.Data[y][x] and returns the radar video processor value
// at the given point. NaN is returned, if no data is available or the
// requested point is located outside the scanned area.
func (c *Composite) At(x, y int) float32 {
180
181
182
	return c.AtZ(x, y, 0)
}

183
184
185
186
// AtZ is shorthand for c.DataZ[z][y][x] and returns the radar video processor
// value at the given point. NaN is returned, if no data is available or the
// requested point is located outside the scanned volume.
func (c *Composite) AtZ(x, y, z int) float32 {
187
	if x < 0 || y < 0 || z < 0 || x >= c.Dx || y >= c.Dy || z >= c.Dz {
188
		return NaN
Jonny Schäfer's avatar
Jonny Schäfer committed
189
	}
190
	return c.DataZ[z][y][x]
Jonny Schäfer's avatar
Jonny Schäfer committed
191
192
193
194
195
196
}

// newError returns an error indicating the failed function and reason
func newError(function, reason string) error {
	return fmt.Errorf("radolan.%s: %s", function, reason)
}