Home

RACPIT visualization

This notebook demonstrates our simulation, preprocessing and visualization pipeline for radar data. See its source here.

import os
os.chdir("..")

from pathlib import Path
import pandas as pd

import numpy as np
import xarray as xr

from utils.synthesize import synthetic_radar
from utils.preprocess import open_recordings, identify_config, raw2rdm

from utils import radar
from utils import skeletons as sk

from utils.visualization import spec_plot, animation_real_synthetic, animation_spec

from ifxaion.daq import Daq

import matplotlib.pyplot as plt
from IPython.display import HTML

from networks.img_transf import MultiTransformNet

import torch
from torch.autograd import Variable

data_path = Path("/mnt/infineon-radar")

raw_dir = data_path / "daq_x-har"
activity = "5_Walking_Boxing"
path = raw_dir / f"{activity}_converted/recording-2020-01-28_12-37-12"

real_path = data_path / "preprocessed/fixed_size/real"
synth_path = data_path / "preprocessed/fixed_size/synthetic"

itn_config = "D"
itn_recording = "2020-02-05T15:16:08"
itn_path = "models/publication_I.model"

Simulation

Radar data loading

daq = Daq(rec_dir=path)
env = daq.env
recording = daq.radar[2]
rec_config = daq.radar[2].cfg
timestamps = recording.data.index
ts_seconds = timestamps.total_seconds()

config_name = identify_config(rec_config)

rec_config['RadarName'] = rec_config.pop("Name")
rec_config['cfg'] = config_name
rec_config["activity"] = activity

n_samples = rec_config['SamplesPerChirp']
m_chirps = rec_config['ChirpsPerFrame']

print(f"Synthetizing data for configuration {config_name}")

Synthetizing data for configuration II

frame_interval_ms = np.mean((timestamps[1:] - timestamps[:-1]).total_seconds()) * 1e3
duration_sec = (timestamps[-1] - timestamps[0]).total_seconds()
print(f'Mean frame interval:\t{frame_interval_ms} ms')
print(f'Total duration:\t{duration_sec} seconds')

Mean frame interval:	32.7835497835498 ms
Total duration:	121.168 seconds

Data synthesis

Load skeleton data

skeletons = sk.load(path, verbose=True)
sk_interp = sk.interpolate(skeletons, timestamps.total_seconds())
sk_da = sk.to_xarray(sk_interp, timestamps, attrs=env)

Reading skeleton data: 1212 skeletons, in 121.1 second recording

Synthesize raw data from skeleton points using a radar configuration

syntheticData = synthetic_radar(skeletons, rec_config, ts_seconds)

assert syntheticData.shape[-2] == m_chirps, "Number of chirps of synthetic data not correct"
assert syntheticData.shape[-1] == n_samples, "Number of samples per chirp of synthetic data not correct"

smin = syntheticData.min()
smax = syntheticData.max()
snorm = (syntheticData - smin) / (smax - smin)
raw_synth = pd.DataFrame({"Timestamps": timestamps, "NormData": [sn for sn in snorm]}).set_index("Timestamps")

Create synthetic data for configuration II
Creating 3697 frames of synthetic data from provided frameTimes
Synthetic data with shape (3697, 1, 64, 256) successfully created

The result is a DataFrame

raw_synth

	NormData
Timestamps
0 days 00:00:01.018000	[[[0.5049517939718389, 0.5049517939718389, 0.5...
0 days 00:00:01.051000	[[[0.5049517939718389, 0.5049517939718389, 0.5...
0 days 00:00:01.083000	[[[0.5049517939718389, 0.5049517939718389, 0.5...
0 days 00:00:01.116000	[[[0.5049517939718389, 0.5049517939718389, 0.5...
0 days 00:00:01.149000	[[[0.5049517939718389, 0.5049517939718389, 0.5...
...	...
0 days 00:02:02.054000	[[[0.49478443427094787, 0.5703649156450615, 0....
0 days 00:02:02.087000	[[[0.5366295082395269, 0.5334191116785014, 0.4...
0 days 00:02:02.120000	[[[0.5013049066653456, 0.5758148999346251, 0.4...
0 days 00:02:02.153000	[[[0.44517515535040936, 0.45493490037400647, 0...
0 days 00:02:02.186000	[[[0.5175748335333865, 0.5360369932409377, 0.4...

3697 rows × 1 columns

Preprocessing

The raw data is processed and converted to an xarray DataArray

rdm_synth = raw2rdm(raw_synth, rec_config, env, name=f"{activity}-{config_name}")

Data shape: (3697, 64, 128)

The Range Doppler Maps can be converted to dB

rdm_db = xr.apply_ufunc(radar.mag2db, rdm_synth, keep_attrs=True, kwargs={"normalize": True})
rdm_db.assign_attrs(units="dB")

<xarray.DataArray '5_Walking_Boxing-II' (time: 3697, doppler: 64, range: 128)>
array([[[-1980.35627771, -1980.35627771, -1980.35627771, ...,
         -1980.35627771, -1980.35627771, -1980.35627771],
        [-1980.35627771,  -672.91793134,  -723.26159038, ...,
         -1980.35627771, -1980.35627771, -1980.35627771],
        [-1980.35627771,  -666.91844933,  -717.26210837, ...,
         -1980.35627771, -1980.35627771, -1980.35627771],
        ...,
        [-1980.35627771,  -663.43208803,  -713.77574707, ...,
         -1980.35627771, -1980.35627771, -1980.35627771],
        [-1980.35627771,  -666.91844933,  -717.26210837, ...,
         -1980.35627771, -1980.35627771, -1980.35627771],
        [-1980.35627771,  -672.91793134,  -723.26159038, ...,
         -1980.35627771, -1980.35627771, -1980.35627771]],

       [[-1980.35627771, -1980.35627771, -1980.35627771, ...,
         -1980.35627771, -1980.35627771, -1980.35627771],
        [-1980.35627771,  -672.91793134,  -723.26159038, ...,
         -1980.35627771, -1980.35627771, -1980.35627771],
        [-1980.35627771,  -666.91844933,  -717.26210837, ...,
         -1980.35627771, -1980.35627771, -1980.35627771],
...
        [ -121.93231657,  -127.91096736,  -177.53243227, ...,
          -197.39423869,  -197.59512174,  -197.74792954],
        [ -121.91844848,  -127.89711643,  -177.51253201, ...,
          -197.55455471,  -197.735938  ,  -197.86734389],
        [ -121.91008633,  -127.88876697,  -177.49895153, ...,
          -197.68555397,  -197.84490509,  -197.95215161]],

       [[ -120.61313815,  -126.58474106,  -177.75736247, ...,
          -190.18859655,  -190.31457682,  -190.3909812 ],
        [ -120.60914293,  -126.58073923,  -177.76141202, ...,
          -190.22786838,  -190.34163099,  -190.40519191],
        [ -120.59722757,  -126.56882891,  -177.75450738, ...,
          -190.27020969,  -190.37143105,  -190.42253751],
        ...,
        [ -120.57760061,  -126.54929189,  -177.68079141, ...,
          -190.09098292,  -190.25284349,  -190.36628012],
        [ -120.59722757,  -126.56887849,  -177.7167001 , ...,
          -190.12005779,  -190.2701266 ,  -190.37148423],
        [ -120.60914293,  -126.58076415,  -177.7423684 , ...,
          -190.15259905,  -190.29080356,  -190.37962968]]])
Coordinates:
  * time     (time) timedelta64[ns] 00:00:01.018000 ... 00:02:02.186000
  * doppler  (doppler) float64 -5.0 -4.841 -4.683 -4.524 ... 4.683 4.841 5.0
  * range    (range) float64 0.0 0.0378 0.07559 0.1134 ... 4.687 4.724 4.762 4.8
Attributes: (12/25)
    units:             dB
    Type:              SdkRadar
    BoardId:           1002
    Port:              COM45
    ChirpsPerFrame:    64
    SamplesPerChirp:   256
    ...                ...
    location:          Dresden-DE
    room_size:         [7.58, 7.64]
    startTime:         2020-01-28T12:37:28
    stopTime:          2020-01-28T12:39:30
    runTimeSeconds:    122
    software:          {'Name': 'DataRecorder', 'Version': '1.7', 'BuildDate'...

Range & Doppler spectrograms in dB can also be calculated

rdm_abs = np.abs(rdm_synth)
rspect = rdm_abs.sum(dim="doppler").assign_attrs({"long_name": "Range spectrogram", "units": "dB"})
dspect = rdm_abs.sum(dim="range").assign_attrs({"long_name": "Doppler spectrogram", "units": "dB"})
synth_spects = xr.Dataset({"range_spect": rspect, "doppler_spect": dspect}, attrs=rdm_synth.attrs)
synth_spects = xr.apply_ufunc(radar.mag2db, synth_spects, keep_attrs=True, kwargs={"normalize": True})

synth_spects

<xarray.Dataset>
Dimensions:        (doppler: 64, range: 128, time: 3697)
Coordinates:
  * time           (time) timedelta64[ns] 00:00:01.018000 ... 00:02:02.186000
  * range          (range) float64 0.0 0.0378 0.07559 0.1134 ... 4.724 4.762 4.8
  * doppler        (doppler) float64 -5.0 -4.841 -4.683 ... 4.683 4.841 5.0
Data variables:
    range_spect    (time, range) float64 -1.99e+03 -584.5 ... -125.6 -125.7
    doppler_spect  (time, doppler) float64 -1.99e+03 -682.8 ... -73.59 -73.54
Attributes: (12/25)
    units:             Complex amplitude
    Type:              SdkRadar
    BoardId:           1002
    Port:              COM45
    ChirpsPerFrame:    64
    SamplesPerChirp:   256
    ...                ...
    location:          Dresden-DE
    room_size:         [7.58, 7.64]
    startTime:         2020-01-28T12:37:28
    stopTime:          2020-01-28T12:39:30
    runTimeSeconds:    122
    software:          {'Name': 'DataRecorder', 'Version': '1.7', 'BuildDate'...

Data animations

Process range & Doppler information from the real recording and extract a short time slice from it

time_slice = slice("00:00:52", "00:01:07")

skeleton_slice = sk_da.sel(time=time_slice)

rdm_real = raw2rdm(recording.data, rec_config, env, name=f"{activity}-{config_name}")
rdm_rabs = np.abs(rdm_real.sel(time=time_slice))

rng_spect = rdm_rabs.sum(dim="doppler").assign_attrs({"long_name": "Range spectrogram", "units": "dB"})
dopp_spect = rdm_rabs.sum(dim="range").assign_attrs({"long_name": "Doppler spectrogram", "units": "dB"})
real_spects = xr.Dataset({"range_spect": rng_spect, "doppler_spect": dopp_spect}, attrs=rdm_real.attrs)
real_spects = xr.apply_ufunc(radar.mag2db, real_spects, keep_attrs=True, kwargs={"normalize": True})

rdm_real_db = xr.apply_ufunc(radar.mag2db, rdm_rabs, keep_attrs=True, kwargs={"normalize": True})
rdm_synth_db = xr.apply_ufunc(radar.normalize_db, rdm_db.sel(time=time_slice), keep_attrs=True)

Data shape: (3697, 64, 128)

%%capture
anim_rs = animation_real_synthetic(rdm_real_db, rdm_synth_db, skeleton_slice,
                                sensor_loc=rec_config["position"], vmin=-40, notebook=True)
anim_spects = animation_spec(rdm_real_db, real_spects, vmin=-40, notebook=True)

HTML(anim_rs.to_html5_video())

HTML(anim_spects.to_html5_video())

Image transformation

Open recordings as a list of Xarray.Datasets

The example uses lazy load, but in the GPU load=True boosts performance

real_recs = open_recordings(itn_config, real_path, load=False)
synth_recs = open_recordings(itn_config, synth_path, load=False)

print(f"{len(real_recs)} recordings have been lazy loaded")
i_rec = [i for i, rec in enumerate(real_recs) if rec.date == itn_recording][0]
print(f"Recording {itn_recording} found at index {i_rec}")

69 recordings have been lazy loaded
Recording 2020-02-05T15:16:08 found at index 67

real_rec = real_recs[i_rec]
synth_rec = synth_recs[i_rec]

Extract short spectrograms from the recordings

tslice = slice("00:01:39", None)
time_length = 64

real_spects = real_rec.drop_vars('label').sel(time=tslice).isel(time=slice(0,time_length))
synth_spects = synth_rec.drop_vars('label').sel(time=tslice).isel(time=slice(0,time_length))

real_spects = xr.apply_ufunc(radar.normalize_db, real_spects.load(), keep_attrs=True)
synth_spects = xr.apply_ufunc(radar.normalize_db, synth_spects.load(), keep_attrs=True)

Transform images

Load Image Transformation Network

dtype = torch.FloatTensor

transformer = MultiTransformNet(num_inputs=2, num_channels=1)
transformer.load_state_dict(torch.load(itn_path))
_ = transformer.eval()

Transform real data with the ITN

range_inp = torch.from_numpy(real_spects.range_spect.values[None, None, :, :])
dopp_inp = torch.from_numpy(real_spects.doppler_spect.values[None, None, :, :])
spec_input = [Variable(range_inp, requires_grad=False).type(dtype), Variable(dopp_inp, requires_grad=False).type(dtype)]

range_hat, doppler_hat = transformer(spec_input)
range_trans = range_hat.detach().numpy()
doppler_trans = doppler_hat.detach().numpy()

spectrograms = xr.merge([real_spects.rename_vars(range_spect="range_real", doppler_spect="doppler_real"),
                         synth_spects.rename_vars(range_spect="range_synth", doppler_spect="doppler_synth")],
                        combine_attrs="drop_conflicts")
spectrograms["range_trans"] = (['time', 'range'],  np.squeeze(range_trans), {"units": "dB"})
spectrograms["doppler_trans"] = (['time', 'doppler'],  np.squeeze(doppler_trans), {"units": "dB"})

Plotting

fig, axes = plt.subplots(2, 3, figsize=(12, 8))

spec_plot(spectrograms[["range_real", "range_trans", "range_synth", "doppler_real", "doppler_trans", "doppler_synth"]],
          axes=axes.flatten(), ax_xlabel=axes[-1], vmin=-40, vmax=0, add_colorbar=False)

im = axes[-1][-1].get_images()[0]
fig.align_ylabels(axes[:,0])
_ = [ax.set_ylabel(None) for ax in axes[:,1:].flatten()]

cbar = plt.colorbar(im, ax=axes, orientation="horizontal")
cbar.set_label("Amplitude [dB]")

for title, ax in zip(("Real data $x$", "Transformed data $\widehat{y}$", "Synthetic data $y$"), axes[0]):
     ax.set_title(title)