Data format expected by notebook timeseries visualizations

[scott-smith]

My company is still  in preview mode of the new Dynatrace platform and UI.  I am trying to get familiar with using notebooks and dashboards.  The data I want to display, availability metrics for synthetic monitors, is not yet in Grail so I am using the classic SDK and the metricClient to get the metric data.  What is the expected format for the data, for say a line graph,  so that I can plot multiple monitors availability over time?  An example would be most helpful.

[scott-smith]

For context, here is the JS code that I was using to attempt to format correctly.  Comments in code as to what kind of worked and what did not.

import { metricsClient } from '@dynatrace-sdk/client-classic-environment-v2';
import type { Timeseries } from '@dynatrace/strato-components-preview/charts';

// Entry point for the notebook
export default async function () {
  //'SMARTS_DOWN'
  //'SMARTS_DEGRADED'
  //HTTP_CHECK-C6E2037C672EC3AF
  //HTTP_CHECK-D1D93CAAAC8D8C85
  //HTTP_CHECK-301A4A1D1D711417
  
  //const metrics = await getMetricsByTags('SMARTS_DOWN');  
  const metrics = await getMetricsByMonitorId('HTTP_CHECK-C6E2037C672EC3AF');
  //console.log(metrics);

  // This works for a single monitor dataset
  return formatAsDataFrame(metrics);

  // Saw this format in docs: https://developer.dynatrace.com/reference/design-system/preview/charts/TimeseriesChart/
  // but does not appear to work for most visualizations
  //return formatAsTimeseriesArray(metrics);

  // Just a variation on the one above, returning as a dataFrame with typings
  // does not work for most visualizations
  //return formatAsTimeseriesDataFrame(metrics);
}

function formatAsTimeseriesDataFrame(metrics){
  let timeSeriesArray = formatAsTimeseriesArray(metrics);
  //console.log(timeSeriesArray);
  
    const types = [{
      mappings: {
        availability: { 
          type: 'array',
          types:[{
            mappings: {
              name: {type: 'string'},
              unit: {type: 'string'},
              datapoints: {
                type: 'array',
                types:[{
                  mappings: {
                    start: {type: 'datetime'},
                    end: {type: 'datetime'},
                    value: {type: 'double'}
                  }
                }]
              }
            }
            }]          
          },
        },
        indexRange: [
        0,
        timeSeriesArray.length -1
      ]
  }]

  // Create a new empty data frame for the result
  const dataFrame = {
    availability: timeSeriesArray,
    types: types
  }

  console.log(dataFrame);
  return dataFrame;
}
  
function formatAsTimeseriesArray(metrics){
  const availabilityMetrics = [];

  for(let i=0; i< metrics[0].data.length; i++) {
    //console.log(metrics[0].data[i].dimensions[0]);
    let timeseries = {
      name: metrics[0].data[i].dimensions[0],
      unit: 'percent',
      datapoints: []
    };

     for(let x=0; x< metrics[0].data[i].timestamps.length; x++){
        if(metrics[0].data[i].values[x] !== null){
          const dp = {
              start: metrics[0].data[i].timestamps[x],
              end: metrics[0].data[i].timestamps[x],
              value: metrics[0].data[i].values[x]
          };
          timeseries.datapoints.push(dp);
        }                       
    }

    availabilityMetrics.push(timeseries);   
  }  

  //console.log(availabilityMetrics);
  return availabilityMetrics;
}

function formatAsDataFrame(metrics){
  const types = [{
      mappings: {
        timeframe: { type: 'timeframe' },
        value: {type: 'double'},
        "entity.id": {type: 'string'}
        },
        indexRange: [
        0,
        metrics[0].data[0].timestamps.length - 1
      ]
  }]

  // Create a new empty data frame for the result
  const dataFrame = {
    records: [],
    types: types
  }

  for(let i=0; i< metrics[0].data.length; i++) {
    for(let x=0; x< metrics[0].data[i].timestamps.length; x++)
    {
      if(metrics[0].data[i].values[x] !== null){
        const record = {
          "entity.id": metrics[0].data[i].dimensions[0],
          timeframe: {
              start: metrics[0].data[i].timestamps[x],
              end: metrics[0].data[i].timestamps[x]
            },
          value: metrics[0].data[i].values[x]        
        }
        dataFrame.records.push(record); 
      }                    
    }    
  }

  //console.log(dataFrame);
  return dataFrame;
}

//get a single HTTP monitor's metrics
async function getMetricsByMonitorId(monitorId) {
  //HTTP_CHECK-301A4A1D1D711417 
  const response = await metricsClient.query({
    acceptType: 'application/json; charset=utf-8', 
    entitySelector: 'entityId('+ monitorId +')',
    from: 'now-3d',
    resolution: 'h',
    metricSelector: 'builtin:synthetic.http.availability.location.totalWoMaintenanceWindow'
  });
  
  return response.result;
}

//get multiple HTTP monitor metrics by tag and type
async function getMetricsByTags(availabilityTagValue) {
  //console.log(availabilityTagValue);
  
  const entitySel = 'type(dt.entity.http_check),tag(SENTRY_AVAILABILITY:' + availabilityTagValue + ')';
  const response = await metricsClient.query({
    acceptType: 'application/json; charset=utf-8', 
    entitySelector: entitySel,
    from: 'now-7d',
    resolution: 'h',
    metricSelector: 'builtin:synthetic.http.availability.location.totalWoMaintenanceWindow'
  });

  return response.result;
}

[stefan_eggersto · Accepted solution]

Hi Scott,

with the dataframe example you were already quite close. In general, the visualizations work when you return the same data structure like the Query API (see type QueryResult). The two properties records and types are relevant for a correct visualization.

I tried to come up with a generic code snippet that should work with different metrics, dimensions and resolutions. Please let me know if this snippet produces the expected visualization.

import { metricsClient } from '@dynatrace-sdk/client-classic-environment-v2';

export default async function () {
  const response = await metricsClient.query({
    acceptType: 'application/json; charset=utf-8', 
    from: 'now-7d',
    resolution: 'h',
    metricSelector: 'builtin:synthetic.http.availability.location.totalWoMaintenanceWindow'
  });
  const metrics = response.result;

  const records = convertMetricsToRecords(metrics);

  // extract dimensions from metric response to attach on returned type mapping
  const dimensionKeys = collectDimensionKeys(metrics);
  const dimensionTypes = dimensionKeys.reduce((result, dimension) => ({
    ...result,
    [dimension]: {
      type: "string"
    }
  }), {});
  
  return {
    records,
    types: [{
      mappings: {
        values: {
          type: "array",
          types: [{
            mappings: {
              element: {
                type: "double"
              }
            }
          }]
        },
        ...dimensionTypes,
        timeframe: {
          type: "timeframe"
        },
        interval: {
          type: "duration"
        }
      },
      indexRange: [0, records.length]
    }],
  };
}

const collectDimensionKeys = (metrics) => {
  const dimensionKeys = metrics
    .flatMap(metric => metric.data.flatMap(data => Object.keys(data.dimensionMap)));
  // ensure dimension keys are unique
  return Array.from(new Set(dimensionKeys));
}

const convertMetricDataToRecords = (metricData) =>
    metricData.map(data => ({
      values: data.values,
      timeframe: {
        start: new Date(data.timestamps[0]).toISOString(),
        end: new Date(data.timestamps[data.timestamps.length - 1]).toISOString(),
      },
      // expects to have at least 2 datapoints
      interval: String((data.timestamps[1] - data.timestamps[0]) * 1000 * 1000),
      // add all dimension values as record fields
      ...data.dimensionMap,
    }));

const convertMetricsToRecords = (metrics) =>  
  metrics.flatMap(metric => convertMetricDataToRecords(metric.data));

[scott-smith]

This does appear to work.  I need to dig into it today, but thank you!