/* Prototype stuff for parsing Level-II data */
+/*
+ * TODO: ARGG, the packet sizses are all wrong..
+ * Check sizes of decompressed bzip files
+ */
#include <stdio.h>
#include <stdlib.h>
#include <errno.h>
#include <error.h>
-#include <arpa/inet.h>
+#include <glib.h>
+#include <bzlib.h>
+#include "level2.h"
-struct packet {
- short junk1[6];
- unsigned short size;
- unsigned char id, type;
- unsigned short seq, gen_date;
- unsigned int gen_time;
- unsigned short num_seg, seg;
- unsigned int coll_time;
- unsigned short coll_date, range, angle, radial, rad_status, elev_angle;
- unsigned short elev_num;
- short first_refl, first_dopp;
- unsigned short refl_size, dopp_size;
- unsigned short num_refl_gate, num_dopp_gate, sector;
- float gain;
- unsigned short refl_ptr, vel_ptr, spec_ptr, dopp_res, pattern;
- short junk2[4];
- unsigned short refl_ptr_rda, vel_ptr_rda, spec_ptr_rda, nyquist, atten;
- short thresh;
- short junk3[17];
- unsigned char data[2304];
- float dbz[460];
-};
+level2_packet_t **level2_split_packets(char *data, int *num_packets, int data_size)
+{
+ int packet_i = 0;
+ level2_packet_t *packet = (level2_packet_t *)data;
+ int max_packets = 128;
+ level2_packet_t **packets = malloc(sizeof(level2_packet_t *)*max_packets);
+ while ((void *)packet < (void *)(data+data_size)) {
+ /* Increase packets size if necessasairy */
+ if (packet_i >= max_packets) {
+ max_packets *= 2;
+ packets = realloc(packets, sizeof(level2_packet_t *)*max_packets);
+ }
-typedef struct {
- /* 24 bytes */
- char version[4];
- char unknown0[16];
- char station[4];
-} __attribute__ ((packed)) header_t;
+ /* Fix byte order for packet */
+ packet->size = g_ntohs(packet->size);
+ packet->seq = g_ntohs(packet->seq);
+ // TODO: Convert the rest of the bytes
-int main(int argc, char **argv)
-{
- if (argc < 2) {
- printf("usage: %s <level2-data>\n", argv[0]);
- return 0;
+ /* Save packet location */
+ packets[packet_i] = packet;
+
+ /* Increment packet and packet_i */
+ // new = old + CTM + 2*size + fcs
+ //packet = (level2_packet_t *)( ((char *)packet) + 12 + 2*packet->size + 4 );
+ packet++;
+ packet_i++;
}
+ packets = realloc(packets, sizeof(level2_packet_t *)*packet_i);
+ *num_packets = packet_i;
+ return packets;
+}
+level2_packet_t *level2_decompress(char *raw_data, int *num_packets)
+{
/* Read header */
- FILE *fd = fopen(argv[1], "r");
+ FILE *fd = fopen(raw_data, "r");
if (fd == NULL)
- error(1, errno, "Error opening files `%s'", argv[1]);
- header_t header;
- if (1 != fread(&header, sizeof(header_t), 1, fd))
+ error(1, errno, "Error opening files `%s'", raw_data);
+ level2_header_t header;
+ if (1 != fread(&header, sizeof(level2_header_t), 1, fd))
error(1, errno, "Error reading header");
- /* Cut up the bzips */
- int file_num = 0;
- char filename[16];
- char *buf = NULL;
- FILE *outfile = NULL;
- unsigned int size = 0;
-
- while ((int)size >= 0) {
- if (1 != fread(&size, 4, 1, fd))
- break; //error(1, errno, "Error reading size, pos=%x", (unsigned int)ftell(fd));
- size = ntohl(size);
+ /* Decompress the bzips
+ * store the entire sequence starting at data
+ * cur_data is for each individual bzip and is the last chunk of data */
+ char *data = NULL;
+ int data_size = 0; // size of previously decmpressed data
+ char *bz2 = NULL; // temp buf for bzipped data
+ unsigned int _bz2_size = 0;
+ while ((int)_bz2_size >= 0) {
+ if (1 != fread(&_bz2_size, 4, 1, fd))
+ break; //error(1, errno, "Error reading _bz2_size, pos=%x", (unsigned int)ftell(fd));
+ _bz2_size = g_ntohl(_bz2_size);
+ int bz2_size = abs(_bz2_size);
/* Read data */
- ;
- if (NULL == (buf = (char *)realloc(buf, size)))
- error(1, errno, "cannot allocate `%d' bytes for buffer", size);
- if (size != fread(buf, 1, size, fd))
+ if (NULL == (bz2 = (char *)realloc(bz2, bz2_size)))
+ error(1, errno, "cannot allocate `%d' bytes for buffer", bz2_size);
+ if (bz2_size != fread(bz2, 1, bz2_size, fd))
error(1, errno, "error reading from input file");
- /* Decmopress data */
- //char *block = (char *)malloc(8192), *oblock = (char *)malloc(262144);
- //error = BZ2_bzBuffToBuffDecompress(oblock, &olength, block, length, 0, 0);
+ /* Decompress on individual bzip to the end of the sequence */
+ unsigned int cur_data_size = 1<<17;
+ int status = BZ_OUTBUFF_FULL;
+ while (status == BZ_OUTBUFF_FULL) {
+ cur_data_size *= 2;
+ data = realloc(data, data_size + cur_data_size);
+ status = BZ2_bzBuffToBuffDecompress(data + data_size, &cur_data_size, bz2, bz2_size, 0, 0);
+ }
+ data_size += cur_data_size; // Add current chunk to decompressed data
- /* Write data */
- snprintf(filename, 16, "%d.bz2", file_num);
- outfile = fopen(filename, "w+");
- fwrite(buf, 1, size, outfile);
- fclose(outfile);
+ /* Debug */
+ //printf("data_size = %d, cur_data_size = %d\n", data_size, cur_data_size);
+ }
+ data = realloc(data, data_size); // free unused space at the end
+
+ return (level2_packet_t *)data;
+ //return level2_split_packets(data, num_packets, data_size);
+}
+
+int main(int argc, char **argv)
+{
+ if (argc < 2) {
+ printf("usage: %s <level2-data>\n", argv[0]);
+ return 0;
+ }
- //fprintf(stderr, "wrote `%d' bytes to file `%s'\n", size, filename);
+ int num_packets;
+ level2_packet_t *packets = level2_decompress(argv[1], &num_packets);
+ printf("read %d packets\n", num_packets);
+
+ //FILE *output = fopen("output.dat", "w+");
+ //fwrite(packets[i], 1, 18470816, output);
+ //fclose(output);
- /* iterate */
- file_num++;
+ FILE *output = fopen("output.dat", "w+");
+ int i;
+ for (i = 0; i < 10000; i++) {
+ printf("packet: size=%x, seq=%d\n", g_ntohs(packets[i].size), g_ntohs(packets[i].seq));
+ fwrite("################", 1, 16, output);
+ fwrite(packets+i, 1, sizeof(level2_packet_t), output);
}
+ fclose(output);
return 0;
}
--- /dev/null
+#ifndef LEVEL2_H
+#define LEVEL2_H
+
+/* http://www.ncdc.noaa.gov/oa/radar/leveliidoc.html */
+typedef struct {
+ /* 24 bytes */
+ char version[4];
+ char unknown0[16];
+ char station[4];
+} __attribute__ ((packed)) level2_header_t;
+
+typedef struct {
+ /**
+ * Channel Terminal Manager:
+ *
+ * Archive II (the data tape) is a copy of messages or data packets
+ * prepared for transmission from the RDA to the RPG. CTM information
+ * is attached to a message or data packet for checking data integrity
+ * during the transmission process and is of no importance to the base
+ * data (omit or read past these bytes).
+ */
+ short ctm[6]; // ignore
+
+ /**
+ * Message Header:
+ *
+ * This information is used to identify either base data or one of
+ * thirteen types of messages that may follow in bytes 28 - 2431. This
+ * header includes the information indicated below:
+ */
+ // Message size in halfwords measure from this halfword to the end of
+ // record.
+ unsigned short size;
+
+ // Channel ID:
+ // 0 = Non-Redundant Site
+ // 1 = Redundant Site Channel 1
+ // 2 = Redundant Site Channel 2
+ unsigned char id;
+
+ // Message type, where:
+ // 1 = DIGITAL RADAR DATA (This message may contain a combination of
+ // either reflectivity, aliased velocity, or spectrum width)
+ // 2 = RDA STATUS DATA.
+ // 3 = PERFORMANCE/MAINTENANCE DATA.
+ // 4 = CONSOLE MESSAGE - RDA TO RPG.
+ // 5 = MAINTENANCE LOG DATA.
+ // 6 = RDA CONTROL COMMANDS.
+ // 7 = VOLUME COVERAGE PATTERN.
+ // 8 = CLUTTER CENSOR ZONES.
+ // 9 = REQUEST FOR DATA.
+ // 10 = CONSOLE MESSAGE - RPG TO RDA.
+ // 11 = LOOP BACK TEST - RDA TO RPG.
+ // 12 = LOOP BACK TEST - RPG TO RDA.
+ // 13 = CLUTTER FILTER BYPASS MAP - RDA to RPG.
+ // 14 = EDITED CLUTTER FILTER BYPASS MAP - RPG to RDA.
+ unsigned char type;
+
+ // I.D. Sequence = 0 to 7FFF, then roll over back to 0.
+ unsigned short seq;
+
+ // Modified Julian date starting from 1/1/70
+ unsigned short gen_date;
+
+ // Generation time of messages in milliseconds of day past midnight
+ // (UTC). This time may be different than time listed in halfwords
+ // 15-16 defined below.
+ unsigned int gen_time;
+
+ // Number of message segments. Messages larger than message size
+ // (halfword 7 defined above) are segmented and recorded in separate
+ // data packets.
+ unsigned short num_seg;
+
+ // Message segment number.
+ unsigned short seg;
+
+ /**
+ * Digital Radar Data Header:
+ *
+ * This information describes the date, time, azimuth, elevation, and
+ * type of base data included in the radial. This header includes the
+ * following information:
+ */
+ // Collection time for this radial in milliseconds of the day from
+ // midnight (UTC).
+ unsigned int coll_time;
+
+ // Modified Julian date referenced from 1/1/70.
+ unsigned short coll_date;
+
+ // Unambiguous range (scaled: Value/10. = KM).
+ unsigned short range;
+
+ // Azimuth angle (coded: [Value/8.]*[180./4096.] = DEG). An azimuth of
+ // "0 degrees" points to true north while "90 degrees" points east.
+ // Rotation is always clockwise as viewed from above the radar.
+ unsigned short angle;
+
+ // Radial number within the elevation scan.
+ unsigned short radial;
+
+ // Radial status where:
+ // 0 = START OF NEW ELEVATION.
+ // 1 = INTERMEDIATE RADIAL.
+ // 2 = END OF ELEVATION.
+ // 3 = BEGINNING OF VOLUME SCAN.
+ // 4 = END OF VOLUME SCAN.
+ unsigned short rad_status;
+
+ // Elevation angle (coded:[Value/8.]*[180./4096.] = DEG). An elevation
+ // of "0 degree" is parallel to the pedestal base while "90 degrees" is
+ // perpendicular to the pedestal base.
+ unsigned short elev_angle;
+
+ // RDA elevation number within the volume scan.
+ unsigned short elev_num;
+
+ // Range to first gate of reflectivity data (METERS). Range may be
+ // negative to account for system delays in transmitter and/or receiver
+ // components.
+ short first_refl;
+
+ // Range to first gate of Doppler data. Doppler data - velocity and
+ // spectrum width (METERS). Range may be negative to account for
+ // system delays in transmitter and/or receiver components.
+ short first_dopp;
+
+ // Reflectivity data gate size (METERS).
+ unsigned short refl_size;
+
+ // Doppler data gate size (METERS).
+ unsigned short dopp_size;
+
+ // Number of reflectivity gates.
+ unsigned short num_refl_gate;
+
+ // Number of velocity and/or spectrum width data gates.
+ unsigned short num_dopp_gate;
+
+ // Sector number within cut.
+ unsigned short sector;
+
+ // System gain calibration constant (dB biased).
+ float gain;
+
+ // Reflectivity data pointer (byte # from the start of digital radar
+ // data message header). This pointer locates the beginning of
+ // reflectivity data.
+ unsigned short refl_ptr;
+
+ // Velocity data pointer (byte # from the start of digital radar data
+ // message header). This pointer locates beginning of velocity data.
+ unsigned short vel_ptr;
+
+ // Spectrum-width pointer (byte # from the start of digital radar data
+ // message header). This pointer locates beginning of spectrum-width
+ // data.
+ unsigned short spec_ptr;
+
+ // Doppler velocity resolution.
+ // Value of: 2 = 0.5 m/s
+ // 4 = 1.0
+ unsigned short dopp_res;
+
+ // Volume coverage pattern.
+ // Value of: 11 = 16 elev. scans/ 5 mins.
+ // 21 = 11 elev. scans/ 6 mins.
+ // 31 = 8 elev. scans/ 10 mins.
+ // 32 = 7 elev. scans/ 10 mins.
+ unsigned short pattern;
+
+ // Unused. Reserved for V&V Simulator.
+ short vv_sim[4];
+
+ // Reflectivity data pointer for Archive II playback. Archive II
+ // playback pointer used exclusively by RDA.
+ unsigned short refl_ptr_rda;
+
+ // Velocity data pointer for Archive II playback. Archive II playback
+ // pointer used exclusively by RDA.
+ unsigned short vel_ptr_rda;
+
+ // Spectrum-width data pointer for Archive II playback. Archive II
+ // playback pointer used exclusively by RDA.
+ unsigned short spec_ptr_rda;
+
+ // Nyquist velocity (scaled: Value/100. = M/S).
+ unsigned short nyquist;
+
+ // Atmospheric attenuation factor (scaled: [Value/1000. = dB/KM]).
+ unsigned short atten;
+
+ // Threshold parameter for minimum difference in echo power between two
+ // resolution volumes for them not to be labeled range ambiguous
+ // (i.e.,overlaid) [Value/10. = Watts].
+ short thresh;
+
+ // Unused.
+ short unused0[17];
+
+ /**
+ * Base Data:
+ *
+ * This information includes the three base data moments; reflectivity,
+ * velocity and spectrum width. Depending on the collection method, up
+ * to three base data moments may exist in this section of the packet.
+ * (For this example, only reflectivity is present.) Base data is coded
+ * and placed in a single byte and is archived in the following format:
+ */
+ // A bit confsued by these docs.
+ // Note: Need to check size field to get correct size.
+ unsigned char data[2300];
+
+ // Reflectivity data (0 - 460 gates) (coded: [((Value-2)/2.)-32. =
+ // dBZ], for Value of 0 or 1 see note below).
+ //unsigned char refl[460];
+
+ // Doppler velocity data (coded: for doppler velocity resolution of 0.5
+ // M/S, [((Value-2)/2.)-63.5 = M/S]; for doppler resolution of 1.0 M/S,
+ // [(Value-2)-127.] = M/S], for Value of 0 or 1 see note below), (0 -
+ // 92 gates). Starting data location depends on length of the
+ // reflectivity field, stop location depends on length of the velocity
+ // field. Velocity data is range unambiguous out to 230 KM.
+ //unsigned char vel[920];
+
+ // Doppler spectrum width (coded: [((Value - 2)/2.)-63.5 = M/S], for
+ // Value of 0 or 1 see note below), (0 - 920 gates). Starting data
+ // location depends on length of the reflectivity and velocity fields,
+ // stop location depends on length of the spectrum width field.
+ // Spectrum width is range unambiguous out to 230 KM.
+ //unsigned char spec[920];
+
+ // Four bytes of trailer characters referred to the Frame Check
+ // Sequence (FCS) follow the data. In cases where the three moments are
+ // not all present or the number of gates for each moment have been
+ // reduced, the record is padded out to a constant size of 1216
+ // halfwords (2432 bytes) following the trailer characters.
+ unsigned int fcs;
+} __attribute__ ((packed)) level2_packet_t;
+
+#endif
projects / grits / commitdiff