Parsing JSON file in Python for "Phantom Data Analysis"¶
Written by: Ashok Tiwari, PhD Candidate, Department of Radiology and Physics, University of Iowa,¶
Date: 4/24/2020
Concept: John Sunderland, PhD¶
Revised at: 05/31/2021¶
To make it comprehendible and to speed up the process, you can restart the kernel and re-run the whole code or cells in notebook. To execute all code cells in your notebook, click Run all (right hand facing double arrow head) on the Jupyter notebook toolbar or press Ctrl + Shift + Alt + Enter.¶
In [1]:
import json
import pandas as pd
from pandas import DataFrame
from pandas.io.json import json_normalize
# To make a printed line bold
from IPython.display import Markdown, display
def printmd(string):
display(Markdown(string))
The cells with # sign is the comment, code is self-explanatory.¶
To run this code step by step, click inside each cell then press: (shift + enter), or you can use Run command
Useful tips:¶
- Q. How to download Jupyter notebook and run this code?
- Go to: Link here select Operating system and download it, then install the program, just follow the on-screen instructions
- Then you should be good to go.
- On MAC OS, click on lunchpad, choose Anaocnda Navigator, then find Jupyter Notebook and lunch it
- On Windows, typ Jupyter Notebook in search box near the start menu, then it should start
- After opening an application it should direct you to the browser
Wait: Select a file for analysis using the dialog box¶
In [2]:
import tkinter as tk
from tkinter import filedialog
window = tk.Tk()
window.wm_attributes('-topmost', 1)
window.withdraw() # this supress the tk window, we don't need it
filename = filedialog.askopenfilename(parent=window,
initialdir="",
title="Select A File",
filetypes = (("Text files", "*.txt"), ("JSON files", "*.json"), ("All files", "*")))
# Here, window.wm_attributes('-topmost', 1) and "parent=window" help open the dialog box on top of other windows
print(type(filename))
In [3]:
# Load JSON file in .txt file format and print headers info under a key (PETScanHeader) using the keywords in the JSON file
with open(filename) as json_file: # filename is a string (str)
data = json.load(json_file) # data is now a dictionery
#print(type(data)) # to check data type
#print(json.dumps(data, indent=2))
printmd('**-----------------PET SCANNER, PHANTOM AND IMAGE INFORMATION---------------------**')
print("Manufacturer:", data.get("PETScanHeader")['Manufacturer'])
print("Scanner Model:", data.get("PETScanHeader")['ManufacturerModelName'])
print("Reconstruction Method:", data.get("PETScanHeader")['ReconstructionMethod'])
print("Phantom Type:", data.get("PhantomType"))
print("Per Bed Duration:", data.get("PETScanHeader")['ActualFrameDuration'])
print("X, Y Pixel Spacing (mm):", data.get("PETScanHeader")['PixelSpacing'])
print("Slice Thickness (mm):", data.get("PETScanHeader")['SliceThickness'])
print("X, Y array (dim):", data.get("PETScanHeader")['Rows'])
print("Iterations:", data.get("PETScanHeader")['Iterations'])
print("Subset:", data.get("PETScanHeader")['Subsets'])
print("Gaussian (mm):", data.get("PETScanHeader")['GaussianFilter'])
# Print headers info under a key (MetaData)
printmd('**-----------------META DATA INFORMATION---------------------**')
print("SNMMI Phantom Analysis Form:", data.get("MetaData")['SNMMIPhantomAnalysisForm'])
print("Site Contact Name:", data.get("MetaData")['SNMMIPhantomAnalysisForm'])
print("SNMMI Phantom Analysis Form:", data.get("MetaData")['Site Contact Name'])
print("Background Activity Syringe (uCi):", data.get("MetaData")['Syringe 1 activity'])
print("Background Syringe Time:", data.get("MetaData")['Syringe 1 assay time'])
print("Background Syringe Residual Activity (uCi):", data.get("MetaData")['Syringe 1 Residual Activity'])
print("Background Syringe Residual Time:", data.get("MetaData")['Syringe 1 Residual Activity Time'])
print("Background Fill Volume (mL):", data.get("MetaData")['Full phantom Weight (g)'])
print("Sphere Activity Syringe (uCi):", data.get("MetaData")['Syringe 2 activity'])
print("Sphere Syringe Time:", data.get("MetaData")['Syringe 2 assay time'])
print("Sphere Syringe Residual Activity (uCi):", data.get("MetaData")['Syringe 2 Residual Activity'])
print("Sphere Syringe Residual Time:", data.get("MetaData")['Syringe 2 Residual Activity Time'])
print("Sphere Solution Volume (mL):", data.get("MetaData")['Sphere Solution Volume (1000 g)'])
print("Expected Contrast:", data.get("ExpectedContrast"))
print("Software Name:", data.get("SoftwareName"))
print("Software Version:", data.get("SoftwareVersion"))
print("Software Date:", data.get("SoftwareDate"))
In [4]:
# Pull out spehere ID's and diameters: using (software in-buit) Model key
# json_normalize is faster than 'for' statement
pd.set_option('display.max_columns', None)
#print("************** SPHERE ID, MODEL POSITION, AND DIAMETER **************")
#model_id = pd.json_normalize(data['Model'])
# It looks like the recent version of json file contains 'MeasurementModel' key instead of 'Model'
model_id = pd.json_normalize(data['MeasurementModel'])
# model_id
# Let us round position values to 4 decimal places
# Create the dataframe
df = pd.DataFrame.from_dict(model_id)
# Flatten Positions with pd.series
df_p = df['Position'].apply(pd.Series)
df_p_0 = df_p[0].apply(pd.Series)
df_p_1 = df_p[1].apply(pd.Series)
df_p_2 = df_p[2].apply(pd.Series)
# Round position values to 4 decimal places
df_p_0 = round(df_p_0, 4)
df_p_1 = round(df_p_1, 4)
df_p_2 = round(df_p_2, 4)
# Rename Position[X, Y, Z] as:
df_p_0.columns = ['Pos_X']
df_p_1.columns = ['Pos_Y']
df_p_2.columns = ['Pos_Z']
df_new = pd.concat([df, df_p_0, df_p_1, df_p_2], axis=1)
# Drop the old column Position and visualize it
df_new = df_new.drop(columns=['Position'])
# Re-arrange columns
df_new = df_new[['ModelPointId', 'Pos_X', 'Pos_Y', 'Pos_Z', 'Diameter', 'Type']]
printmd('**Table 1: Pos_X, Pos_Y and Pos_Z are centroids of the Model spheres**')
df_new
Out[4]:
In [5]:
# Pull out phantom measurement spehere ID's and diameters: using MeasurementModel key (Transformed Position)
printmd('**Table 2: SPHERE ID, MEASUREMENT POSITION, DIAMETER, AND INSERT TYPE**')
measurement_ids = pd.json_normalize(data, "MeasurementModel")
#measurement_ids
# Create the dataframe
ef = pd.DataFrame.from_dict(measurement_ids)
# Flatten Positions with pd.series
ef_p = ef['Position'].apply(pd.Series)
ef_p_0 = ef_p[0].apply(pd.Series)
ef_p_1 = ef_p[1].apply(pd.Series)
ef_p_2 = ef_p[2].apply(pd.Series)
# Round position values to 4 decimal places
ef_p_0 = round(ef_p_0, 4)
ef_p_1 = round(ef_p_1, 4)
ef_p_2 = round(ef_p_2, 4)
# Rename Position[X, Y, Z] as:
ef_p_0.columns = ['Pos_X']
ef_p_1.columns = ['Pos_Y']
ef_p_2.columns = ['Pos_Z']
ef_new = pd.concat([ef, ef_p_0, ef_p_1, ef_p_2], axis=1)
# Drop the old column Position and visualize it
ef_new = ef_new.drop(columns=['Position'])
# Re-arrange columns
ef_new = ef_new[['ModelPointId', 'Pos_X', 'Pos_Y', 'Pos_Z', 'Diameter', 'Type']]
print('**Note: Pos_X, Pos_Y and Pos_Z are centroids of the spheres**')
ef_new
Out[5]:
In [6]:
# Let us check the JSON phantom key "Measurements"
measurements = pd.json_normalize(data['Measurements'])
# let us check the index, columns and values of pandas Dataframe "measurements"
index = measurements.index
columns = measurements.columns
values = measurements.values
# columns # to pull out Pull out columns headers name
# Let us set rows and columns of the data to be visualize, set each as 500. The default value is 10, i.e, if no number of
# rows or columns are specified it will print only 10 rows
pd.set_option('display.max_rows', 500)
pd.set_option('display.max_columns', 500)
# Let's select all the columns -- ModelPointId, Diameter, IsInsert, Mean, etc. (see below)
# Let us read all data values in measurements; it is the block of all data values for all spheres.
# We can pull out the data for individual spheres; see below for each spheres
# let us round the data for 4 decimal place, data [] are not numbers so special treatment should be done if we are interested!
# If you want to visualize all the data then uncomment following two lines
#print("********************* COLLECTIVE TABLE FOR SPHERE INSERTS AND BACKGROUND ROI'S FOR ALL DIAMETERS ************************")
#round(measurements[['ModelPointId', 'Center', 'Diameter', 'IsInsert', 'Mean', 'StdDev', 'Max', 'Peak', 'Peak500', 'PeakLocation', 'Peak500Location']], 4)
In [7]:
# Data belong to sphere size 37 mm
# First column is the index
pd.set_option('display.max_rows', None) # None is unlimited, default 10 rows
pd.set_option('display.max_columns', None)
df = round(measurements[['ModelPointId', 'Diameter', 'IsInsert', 'Mean', 'StdDev', 'Max', 'Peak', 'Peak500']], 4)
# Select rows where the diameter of the sphere is 37.0 mm
printmd('**Table 3: STATISTICS OF 37 MM DIAMETER SPHERE**')
select_37mm = df.loc[(df['Diameter'] == 37.0) & (df['IsInsert'] == False)]
select_37mm
Out[7]:
In [8]:
pd.set_option('display.max_rows', None) # None is unlimited, default 10 rows
pd.set_option('display.max_columns', None)
df = round(measurements[['ModelPointId', 'Diameter', 'IsInsert', 'Mean', 'StdDev', 'Max', 'Peak', 'Peak500']], 4)
# Select rows where the diameter of the sphere is 28.0 mm
printmd('**Table 4: STATISTICS OF 28 MM DIAMETER SPHERE**')
select_28mm = df.loc[(df['Diameter'] == 28.0) & (df['IsInsert'] == False)]
select_28mm
Out[8]:
In [9]:
pd.set_option('display.max_rows', None) # None is unlimited, default 10 rows
pd.set_option('display.max_columns', None)
df = round(measurements[['ModelPointId', 'Diameter', 'IsInsert', 'Mean', 'StdDev', 'Max', 'Peak', 'Peak500']], 4)
# Select rows where the diameter of the sphere is 22.0 mm
printmd('**Table 5: STATISTICS OF 22 MM DIAMETER SPHERE**')
select_22mm = df.loc[(df['Diameter'] == 22.0) & (df['IsInsert'] == False)]
select_22mm
Out[9]:
In [10]:
df = round(measurements[['ModelPointId', 'Diameter', 'IsInsert', 'Mean', 'StdDev', 'Max', 'Peak', 'Peak500']], 4)
# Select rows where the diameter of the sphere is 17.0 mm
printmd('**Table 6: STATISTICS OF 17 MM DIAMETER SPHERE**')
select_17mm = df.loc[(df['Diameter'] == 17.0) & (df['IsInsert'] == False)]
select_17mm
Out[10]:
In [11]:
df = round(measurements[['ModelPointId', 'Diameter', 'IsInsert', 'Mean', 'StdDev', 'Max', 'Peak', 'Peak500']], 4)
# Select rows where the diameter of the sphere is 13.0 mm
printmd('**Table 7: STATISTICS OF 13 MM DIAMETER SPHERE**')
select_13mm = df.loc[(df['Diameter'] == 13.0) & (df['IsInsert'] == False)]
select_13mm
Out[11]:
In [12]:
df = round(measurements[['ModelPointId', 'Diameter', 'IsInsert', 'Mean', 'StdDev', 'Max', 'Peak', 'Peak500']], 4)
# Select rows where the diameter of the sphere is 13.0 mm
printmd('**Table 8: STATISTICS OF 10 MM DIAMETER SPHERE**')
select_10mm = df.loc[(df['Diameter'] == 10.0) & (df['IsInsert'] == False)]
select_10mm
Out[12]:
In [13]:
df = round(measurements[['ModelPointId', 'Diameter', 'IsInsert', 'Mean', 'StdDev', 'Max', 'Peak', 'Peak500']], 4)
# Select rows where the diameter of the sphere is 13.0 mm
printmd('**Table 9: STATISTICS OF 7 MM DIAMETER SPHERE**')
select_7mm = df.loc[(df['Diameter'] == 7.0) & (df['IsInsert'] == False)]
select_7mm
Out[13]:
In [14]:
# background mean per sphere diameter
BackgroundMeanPerDiameter = pd.json_normalize(data['BackgroundMeanPerDiameter'])
ContrastRecoveryToNoiseRatios = pd.json_normalize(data['ContrastRecoveryToNoiseRatios'])
BackgroundMeanPerDiameter = BackgroundMeanPerDiameter.rename(index={0: "RCMean"})
BackgroundMeanPerDiameter = BackgroundMeanPerDiameter.transpose()
ContrastRecoveryToNoiseRatios = ContrastRecoveryToNoiseRatios.rename(index={0: "ContrastRecoveryToNoiseRatio"})
ContrastRecoveryToNoiseRatios = ContrastRecoveryToNoiseRatios.transpose()
output = pd.concat([BackgroundMeanPerDiameter, ContrastRecoveryToNoiseRatios], axis=1, sort=False)
printmd('**Table 10: RC mean & Contrast Recovery To Noise Ratio**')
output
Out[14]:
In [15]:
pd.set_option('display.max_rows', None)
pd.set_option('display.max_columns', None)
A = pd.json_normalize(data['RecoveryCoefficentsMean'])
B = pd.json_normalize(data['ContrastRecoveryCoefficentsMean'])
C = pd.json_normalize(data['RecoveryCoefficentsMax'])
D = pd.json_normalize(data['ContrastRecoveryCoefficentsMax'])
E = pd.json_normalize(data['RecoveryCoefficentsPeak'])
F = pd.json_normalize(data['ContrastRecoveryCoefficentsPeak'])
G = pd.json_normalize(data['RecoveryCoefficentsPeak500'])
H = pd.json_normalize(data['ContrastRecoveryCoefficentsPeak500'])
A = A.rename(index={0: "RCMean"})
A = A.transpose()
B = B.rename(index={0: "CRCMean"})
B = B.transpose()
C = C.rename(index={0: "RCMax"})
C = C.transpose()
D = D.rename(index={0: "CRCMax"})
D = D.transpose()
E = E.rename(index={0: "RCPeak"})
E = E.transpose()
F = F.rename(index={0: "CRCPeak"})
F = F.transpose()
G = G.rename(index={0: "RCPeak500"})
G = G.transpose()
H = H.rename(index={0: "CRCPeak500"})
H = H.transpose()
result = pd.concat([A, B, C, D, E, F, G, H], axis=1, sort=False)
printmd('**Table 11: STATISTICS IN TABULAR FORM**')
printmd('Note: Upper 37-7 mm sphere data in table are for uniform background, and lower 22-10 mm for lung background')
# Add diameter (mm) information
# Indexes 0, 1, 2, ....11 are respectively 37, 28, 22, 17, 13, 10, 7, 22, 17, 13, 10, 10 mm spheres
# Rename the index numbers by spheres diameters
result.index = [37, 28, 22, 17, 13, 10, 7, 22, 17, 13, 10, 10]
result
Out[15]:
In [16]:
import matplotlib.pyplot as plt
%matplotlib inline
plt.figure()
plt.figure(figsize=(8,5))
x = 37, 28, 22, 17, 13, 10
y = A[0:6] # A is Recovery Ceofficient Mean from above table
y1 = B[0:6]
y2 = C[0:6]
y3 = D[0:6]
y4 = E[0:6]
y5 = F[0:6]
y6 = G[0:6]
y7 = H[0:6]
plt.xticks(fontsize=16)
plt.yticks(fontsize=16)
plt.xlabel('Sphere Diameter (mm)',fontsize=20)
plt.ylabel('Statistics',fontsize=20)
plt.plot(x, y, label='RCMean', color='red', linestyle='dashed', marker='o', markersize=10)
plt.plot(x, y1, label='CRCMean', color='blue', linestyle='dashed', marker='v', markersize=10)
plt.plot(x, y2, label='RCMax', color='yellow', linestyle='dashed', marker='^', markersize=10)
plt.plot(x, y3, label='CRCMax', color='black', linestyle='dashed', marker='8', markersize=10)
plt.plot(x, y4, label='RCPeak', color='grey', linestyle='dashed', marker='p', markersize=10)
plt.plot(x, y5, label='CRCPeak', color='brown', linestyle='dashed', marker='P', markersize=10)
plt.plot(x, y6, label='RCPeak500', color='green', linestyle='dashed', marker='H', markersize=10)
plt.plot(x, y7, label='CRCPeak500', color='magenta', linestyle='dashed', marker='+', markersize=10)
plt.legend(loc='lower right',prop={'size': 12})
#plt.title('Statistics vs. sphere diameter', fontsize=16 )
#plt.grid(True)
Out[16]:
