Refactor time-series code
Dependencies
- [2]
NMT3JJPPAdd function to find median - [3]
2AE4VE7UAdd support for 2001 census - [4]
QU7MDQEDAdd basic support for 1996 census data - [5]
4YKXEBAVReduce inconsistencies between 1996 and 2001 objects - [6]
QDEBWR2RAdd missing state time series data - [7]
YXYO3B7JRestructure filesystem - [8]
FCHAPZLDImplement simpler file merging function - [9]
7ZN6HHL2Extract relevant census data
Change contents
- edit in project.py at line 80
# TODO: assert table is rectangular - edit in project.py at line 101
# Simple function that says whether or not to use a categorydef should_keep(category):# If any of these phrases are in the category, it's not allowed!bad_words = ["Neg_Ni_inc", # Negative/nil income"more", # Brackets over a certain threshold (eg $4000more)"or more","NS", # Not stated"Notstated","Tot", # Total"PI_S",]for bad_word in bad_words:if bad_word in category:# Bad word! Tell them not to use this categoryreturn False# Made it to the end without any bad words, should be finereturn True# Take a valid category and return the year, mean income & rentdef split_category(category):# Example category: 'C11_1_149_R1_74'parts = category.split("_")# Make sure it's a valid categoryif not should_keep(category):print(f"Invalid category: {category}")assert should_keep(category) is True# All valid categories have 5 partsassert len(parts) == 5# Split up the partsyear = parts[0]min_income = int(parts[1])max_income = int(parts[2])max_rent = int(parts[4])# Year starts with 'C' followed by 2 digitsassert len(year) == 3assert year[0] == "C"assert year[1:].isdigit()# `min_rent` is special because it has the 'R' prefixmin_rent = parts[3]# Make sure the first character is an 'R'assert min_rent[0] == "R"# Then we can just convert it into an int like the othersmin_rent = int(min_rent[1:])mean_income = (min_income + max_income + 1) // 2mean_rent = (min_rent + max_rent + 1) // 2return (year, mean_income, mean_rent) - replacement in project.py at line 335
def merge_files(files):# Set up our variablescategories = []years = {}class TimeSeriesBase(CensusDataset):def __init__(self, dataset):self.categories = dataset[0]self.data = dataset[1:] - replacement in project.py at line 340
for filename in files:file = open(filename, "r").readlines()# Split each row on the commafile = [[col.strip() for col in row.split(",")] for row in file]# First line in file (file[0]) is headingsfirst_line = file[0]# Don't need the first column, as it says 'STE_CODE_2021'# Example value for `categories_for_file`:# ['C11_Neg_Ni_inc_R1_74', 'C11_Neg_Ni_inc_R75_99', 'C11_Neg_Ni_inc_R100_149', etc]categories_for_file = first_line[1:]# When we need to use categories, use `categories_for_file`# The `file` variable should be JUST data, so remove the first line# Example value:"""[['1', '400', '505', etc],['2', '241', '297', etc],etc]"""file = file[1:]def median_incomes(self):incomes = {}for column in range(len(self.categories)):category = self.categories[column] - replacement in project.py at line 345
# Get a list of all the areas (states, could later be LGAs)# Should end up looking like:# ['1', '2', '3', etc]areas = []for row in range(len(file)):# Area code is the first column of each row# Example value: '1'first_cell = file[row][0]# Add that to list of all areasareas.append(first_cell)total_income = 0for row in range(len(self.data)):total_income += int(self.data[row][column]) - replacement in project.py at line 349
# Remove this column from the rowfile[row] = file[row][1:]income = category[1]if income not in incomes:incomes[income] = 0incomes[income] += total_income - replacement in project.py at line 354
# Go through ALL the data in the file column-by-columnfor column in range(len(categories_for_file)):# Example category: 'C11_Neg_Ni_inc_R1_74'category = categories_for_file[column]return incomes - replacement in project.py at line 356
# Example value: 'C11_Neg_Ni_inc_R1_74'# Split the category up into parts (separated by underscores)category = category.split("_")# Example value for category: ['C11', 'Neg', 'Ni', etc]# Year is always prefixed, so get everything before first underscore# Example value: 'C11' (first item in category)year = category[0]# Category is everything after the first underscore# Example value: 'Neg_Ni_inc_R1_74'category = "_".join(category[1:])def median_rents(self):rents = {}for column in range(len(self.categories)):category = self.categories[column] - replacement in project.py at line 361
# Add that category to the list of categories (without year)# Example value: 'Neg_Ni_inc_R1_74' (no year included!!)# TODO: assert all categories are the same across yearscategories.append(category)total_rent = 0for row in range(len(self.data)):total_rent += int(self.data[row][column]) - replacement in project.py at line 365
# Set up the year if none exists# Example year: 'C11'if year not in years:years[year] = {}# Should look like:"""{'C11': {},}"""rent = category[0]if rent not in rents:rents[rent] = 0rents[rent] += total_rent - replacement in project.py at line 370
# Now add every row in this columnfor row in range(len(file)):# years[year] is a dict mapping states to rows# Every key is essentially one row of data in the csv# It will end up looking something like:"""{'1': [1, 2, 3, etc],'2': [3, 1, 2, etc],etc}"""# Example value: first row (row = 0), area = '1'area = areas[row]return rents - edit in project.py at line 372
# If the area hasn't been seen beforeif area not in years[year]:# Add an empty listyears[year][area] = [] - replacement in project.py at line 373
# This is the very big long row for the current yearcurrent_cell = file[row][column]# Add the current cell to the relevant datasetyears[year][area].append(current_cell)class Census2011(TimeSeriesBase):def year(self):return 2011 - edit in project.py at line 377
return (categories, years) - edit in project.py at line 378
class Census2016(TimeSeriesBase):def year(self):return 2016 - edit in project.py at line 382[3.4032]→[3.4032:4189](∅→∅),[3.4189]→[3.5785:5960](∅→∅),[3.5960]→[3.4269:4861](∅→∅),[3.4269]→[3.4269:4861](∅→∅)
# Simple function that says whether or not to use a categorydef should_keep(category):# If any of these phrases are in the category, it's not allowed!bad_words = ["Neg_Ni_inc","4000more","R650more","R_NS","R_Tot","PI_S","or more","Notstated",]for bad_word in bad_words:if bad_word in category:# Bad word! Tell them not to use this categoryreturn False# Made it to the end without any bad words, should be finereturn True# Take a valid category and return the mean income & rentdef split_category(category):# Example category: '1_149_R1_74'parts = category.split("_")# Make sure it's a valid categoryif not should_keep(category):print(category)assert should_keep(category) is True# All valid categories have 4 partsassert len(parts) == 4 - replacement in project.py at line 383
# Split up the partsmin_income = int(parts[0])max_income = int(parts[1])max_rent = int(parts[3])class Census2021(TimeSeriesBase):def year(self):return 2021 - edit in project.py at line 387
# `min_rent` is special because it has the 'R' prefixmin_rent = parts[2]# Make sure the first character is an 'R'assert min_rent[0] == "R"# Then we can just convert it into an int like the othersmin_rent = int(min_rent[1:]) - replacement in project.py at line 388
mean_income = round((min_income + max_income) / 2)mean_rent = round((min_rent + max_rent) / 2)return (mean_income, mean_rent)# Second step: split 2-variable categories into tabledef reshape_into_table(categories, years):# Example value: 'C11'for year in years:# Example value: '1'for area in years[year]:# TODO: categories may change between iterations# This should look like:"""[# This is income of $1-$149[# This is rent of $1-$741535,# This is rent of $75-$991580,etc],# This is income of $150-$299[# This is rent of $1-$744591,# This is rent of $75-$992188,etc],etc]"""new_data = []# The last mean income we sawlast_income = None# Make sure there are as many categories as dataassert len(years[year][area]) == len(categories)# Go through each 2-variable category and data pairfor column in range(len(categories)):# Get the relevant categorycategory = categories[column]# Check if the category is part of our datasetif should_keep(category) is True:income, rent = split_category(category)class Census2011_2016_2021(CensusDataset):def __init__(self, files):census2011 = Nonecensus2016 = Nonecensus2021 = None - replacement in project.py at line 394
# Check if we've hit a new income bracketif last_income != income:new_data.append([])last_income = incomefor file_index in range(len(files)):file = open(files[file_index], "r").read()file = tabulate(file) - replacement in project.py at line 398
# Append our rent information to the most recent bracket# The most recent bracket will always be the one we wantcurrent_cell = years[year][area][column]current_cell = int(current_cell)new_data[-1].append(current_cell)# First-time setup requires opening the first fileif file_index == 0:census2011 = [[] for row in range(len(file))]census2016 = [[] for row in range(len(file))]census2021 = [[] for row in range(len(file))] - replacement in project.py at line 404
# Update this from old->new datayears[year][area] = new_data# Headings are the first rowheadings = file[0]# Make sure to skip the first column (STE_CODE_2021)for column in range(1, len(file[0])):if should_keep(headings[column]):(year, mean_income, mean_rent) = split_category(headings[column]) - replacement in project.py at line 411
incomes = []rents = []for category in categories:if should_keep(category):(mean_income, mean_rent) = split_category(category)if mean_income not in incomes:incomes.append(mean_income)if mean_rent not in rents:rents.append(mean_rent)# Add each cell to its corresponding rowfor row in range(len(file)):if row == 0:cell = (mean_income, mean_rent)else:cell = file[row][column] - replacement in project.py at line 418
return ((incomes, rents), years)if year == "C11":census2011[row].append(cell)elif year == "C16":census2016[row].append(cell)elif year == "C21":census2021[row].append(cell)else:raise ValueError("Unsupported census year") - edit in project.py at line 427
# Make sure headings are consistent across yearsassert census2011[0] == census2016[0] == census2021[0] - replacement in project.py at line 430
# (categories, years) = merge_files([TIME_SERIES_A])# (categories, years) = reshape_into_table(categories, years)# print(categories)self.census2011 = Census2011(census2011)self.census2016 = Census2016(census2016)self.census2021 = Census2021(census2021) - replacement in project.py at line 434
# parts = merge_time_series_files(STATE_TIME_SERIES)def get_year(self, year):if year == 2011:return self.census2011elif year == 2016:return self.census2016elif year == 2021:return self.census2021else:raise ValueError("Unsupported year") - replacement in project.py at line 503
print(Census1996().median_incomes())print(Census1996().median_rents())print(Census2001().median_incomes())print(Census2001().median_rents())# print(Census1996().median_incomes())# print(Census1996().median_rents())# print(Census2001().median_incomes())# print(Census2001().median_rents()) - replacement in project.py at line 508
census = Census2001().median_incomes()for key in census:print(f"{key},{census[key]}")# census = Census2001().median_incomes()# for key in census:# print(f"{key},{census[key]}")# print(find_median(census)) - replacement in project.py at line 514
print(find_median(census))# print(merge_files(STATE_TIME_SERIES)[0])time_series = Census2011_2016_2021(STATE_TIME_SERIES)print(time_series.get_year(2011).median_rents())