{ "cells": [ { "cell_type": "code", "execution_count": 235, "metadata": {}, "outputs": [], "source": [ "# imports\n", "import pandas as pd\n", "import numpy as np\n", "\n", "from great_schools import get_nearby_schools\n", "from distance import get_distance\n", "from secret import get_key\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Shaun and Daniela's Boston Public School Analysis\n", "#### 2021.04.10" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Fetch the API key from the local filesystem." ] }, { "cell_type": "code", "execution_count": 236, "metadata": {}, "outputs": [], "source": [ "# get the API key\n", "api_key_file = '../keys/api.key'\n", "api_key = get_key(api_key_file)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Use the `nearby_schools` API endpoint to grab raw data of all schools within the maximum radius" ] }, { "cell_type": "code", "execution_count": 237, "metadata": {}, "outputs": [], "source": [ "# Some columns will dropped immediately as pre-processing.\n", "drops = [\n", " 'nces-id',\n", " 'school-summary',\n", " 'street',\n", " 'fipscounty',\n", " 'phone',\n", " 'fax',\n", " 'web-site',\n", " 'overview-url',\n", " 'rating-description',\n", " 'distance',\n", "]\n", "\n", "# Grab data for Boston.\n", "refresh = False\n", "boston_nearby_schools_file = '../data/nearby_schools/boston.csv'\n", "if refresh:\n", " boston_schools = get_nearby_schools(api_key,\"42.3\",\"-71.2\",\"50\")\n", " boston_df = pd.DataFrame.from_dict(boston_schools)\n", " boston_df.drop(columns=drops,inplace=True)\n", " boston_df.to_csv(boston_nearby_schools_file, )\n", "else:\n", " boston_df = pd.read_csv(boston_nearby_schools_file)\n", " boston_df.set_index(keys=[\"universal-id\"], drop=True, inplace=True)\n", " boston_df.drop(columns=[\"Unnamed: 0\"], inplace=True)\n", "\n", "# Grab data for Buffalo.\n", "refresh = False\n", "buffalo_nearby_schools_file = '../data/nearby_schools/buffalo.csv'\n", "if refresh:\n", " buffalo_schools = get_nearby_schools(api_key,\"42.9625\",\"-78.7425\",\"50\")\n", " buffalo_df = pd.DataFrame.from_dict(buffalo_schools)\n", " buffalo_df.drop(columns=drops,inplace=True)\n", " buffalo_df.to_csv(buffalo_nearby_schools_file)\n", "else:\n", " buffalo_df = pd.read_csv(buffalo_nearby_schools_file)\n", " buffalo_df.set_index(keys=[\"universal-id\"], drop=True, inplace=True)\n", " buffalo_df.drop(columns=[\"Unnamed: 0\"], inplace=True)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Process the `lat` and `lon` columns from the API output into tuples.\n", "\n", "Then create two new columns:\n", "- Distance to Downtown\n", "- Distance to Work" ] }, { "cell_type": "code", "execution_count": 238, "metadata": {}, "outputs": [], "source": [ "# Form tuple to represent coordinates\n", "boston_df['coordinates'] = list(zip(boston_df.lat,boston_df.lon))\n", "#boston_df.drop(columns=['lat', 'lon'], inplace=True)\n", "\n", "# Define coordinates of important places\n", "downtown=(42.3674836866797, -71.07134540735377) # Science Museum\n", "work=(42.47381059540949, -71.25414135292398) # Hartwell\n", "\n", "# Create new columns to tabulate distance to these important places\n", "boston_df['distance-to-downtown'] = boston_df['coordinates'].apply(func=get_distance,p2=downtown)\n", "boston_df['distance-to-work'] = boston_df['coordinates'].apply(func=get_distance,p2=work)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "We should definitely removal all schools that aren't in Massachusetts." ] }, { "cell_type": "code", "execution_count": 239, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "There are 1789 schools from the original API results.\n", "Allowing only schools from Massachusetts reduces the dataset to 1375 schools.\n" ] } ], "source": [ "print(f'There are {len(boston_df)} schools from the original API results.')\n", "\n", "# only allow from MA\n", "boston_df = boston_df[boston_df['state'] == \"MA\"]\n", "print(f'Allowing only schools from Massachusetts reduces the dataset to {len(boston_df)} schools.')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "How many unique district id's are there?" ] }, { "cell_type": "code", "execution_count": 240, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "There are 230 unique school districts.\n", "\n" ] } ], "source": [ "# get unique districts\n", "districts = boston_df[\"district-id\"].unique()\n", "print(f'\\nThere are {len(districts)} unique school districts.\\n')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Which of these districts are close to both work and downtown boston?" ] }, { "cell_type": "code", "execution_count": 241, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "There are 90 school districts within reasonable proximity to downtown and work.\n", "\n", "There are 699 schools within these proximal districts.\n", "\n" ] } ], "source": [ "# calculate distance to PoI using geo-center of districts\n", "distances_to_downtown = {k: np.mean(list(v)) for k, v in boston_df.groupby('district-id')['distance-to-downtown']}\n", "distances_to_work = {k: np.mean(list(v)) for k, v in boston_df.groupby('district-id')['distance-to-work']}\n", "\n", "df_downtown = pd.DataFrame.from_dict(distances_to_downtown, orient='index')\n", "df_work = pd.DataFrame.from_dict(distances_to_work, orient='index')\n", "\n", "# merge these new columns\n", "both_df = pd.merge(left=df_downtown, right=df_work, how='inner', left_index=True, right_index=True)\n", "both_df.rename(columns={'0_x': \"downtown\", '0_y': \"work\"}, inplace=True)\n", "\n", "both_df = both_df[both_df[\"downtown\"] < 35.0]\n", "both_df = both_df[both_df[\"work\"] < 20.0]\n", "\n", "print(f'\\nThere are {len(both_df)} school districts within reasonable proximity to downtown and work.\\n')\n", "\n", "# filter out all schools which aren't in proximal districts\n", "proximal_district_ids = list(both_df.index)\n", "boston_df = boston_df[boston_df['district-id'].isin(proximal_district_ids)]\n", "\n", "print(f'There are {len(boston_df)} schools within these proximal districts.\\n')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Let's drop any districts that have an average rating below the school population mean." ] }, { "cell_type": "code", "execution_count": 242, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "Of the remaining 699 schools, the average rating is 5.664546899841017.\n", "\n", "There are 56 districts remaining after pruning districts whose collective average is below the population mean rating.\n", "\n", "Which are, ['Acton-Boxborough School District', 'Andover School District', 'Arlington Public Schools', 'Ashland School District', 'Assabet Valley Regional Vocational Technical School District', 'Bedford School District', 'Belmont School District', 'Billerica School District', 'Boxford School District', 'Brookline School District', 'Burlington School District', 'Cambridge School District', 'Carlisle School District', 'Chelmsford School District', 'Concord School District', 'Concord-Carlisle School District', 'Dover School District', 'Dover-Sherborn School District', 'Dracut School District', 'Essex North Shore Agricultural and Technical School District', 'Groton-Dunstable School District', 'Harvard School District', 'Lexington School District', 'Lincoln-Sudbury School District', 'Littleton School District', 'Lynnfield School District', 'Marblehead School District', 'Masconomet School District', 'Melrose School District', 'Middleton School District', 'Milton School District', 'Nahant School District', 'Nashoba School District', 'Natick School District', 'Needham School District', 'Newton School District', 'North Andover School District', 'North Reading School District', 'Norwood School District', 'Quincy School District', 'Reading School District', 'Shawsheen Valley Regional Vocational Technical School District', 'Sherborn School District', 'Southborough School District', 'Stoneham School District', 'Sudbury School District', 'Topsfield School District', 'Tyngsborough School District', 'Wakefield School District', 'Wayland School District', 'Wellesley School District', 'Westford School District', 'Weston School District', 'Westwood School District', 'Wilmington School District', 'Winchester School District']\n" ] } ], "source": [ "# get the mean rating from the entire population of schools\n", "mean_rating = boston_df['rating'].mean()\n", "std_rating = boston_df['rating'].std()\n", "\n", "print(f'\\nOf the remaining {len(boston_df)} schools, the average rating is {mean_rating}.')\n", "\n", "# compute the average rating for each district\n", "ave_ratings = {k: np.mean(v) for k, v in boston_df.groupby(by='district-id')['rating']}\n", "\n", "# keep only districts that are above the population mean\n", "not_low_performing = [k for k, v in ave_ratings.items() if v > mean_rating]\n", "boston_df = boston_df[boston_df['district-id'].isin(not_low_performing)]\n", "\n", "districts = sorted(list(boston_df['district-name'].unique()))\n", "print(f'\\nThere are {len(districts)} districts remaining after pruning districts whose collective average is below the population mean rating.\\n')\n", "print(f'Which are, {districts}')" ] }, { "cell_type": "code", "execution_count": 243, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
| \n", " | \n", " | state-id | \n", "type | \n", "level-codes | \n", "level | \n", "city | \n", "state | \n", "zip | \n", "county | \n", "lat | \n", "lon | \n", "district-id | \n", "rating | \n", "year | \n", "coordinates | \n", "distance-to-downtown | \n", "distance-to-work | \n", "
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| district-name | \n", "name | \n", "\n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " |
| Newton School District | \n", "Charles E Brown Middle School | \n", "2070310 | \n", "public | \n", "m | \n", "6,7,8 | \n", "Newton Centre | \n", "MA | \n", "2459 | \n", "Middlesex County | \n", "42.308750 | \n", "-71.190292 | \n", "304 | \n", "9.0 | \n", "2021.0 | \n", "(42.30875, -71.190292) | \n", "7.300583 | \n", "11.852619 | \n", "
| Oak Hill Middle School | \n", "2070320 | \n", "public | \n", "m | \n", "6,7,8 | \n", "Newton | \n", "MA | \n", "2459 | \n", "Middlesex County | \n", "42.310009 | \n", "-71.191544 | \n", "304 | \n", "8.0 | \n", "2021.0 | \n", "(42.310009, -71.191544) | \n", "7.306174 | \n", "11.751548 | \n", "|
| Countryside Elementary School | \n", "2070040 | \n", "public | \n", "e | \n", "KG,1,2,3,4,5 | \n", "Newton Highlands | \n", "MA | \n", "2461 | \n", "Middlesex County | \n", "42.313141 | \n", "-71.202377 | \n", "304 | \n", "7.0 | \n", "2021.0 | \n", "(42.313141, -71.202377) | \n", "7.667944 | \n", "11.403174 | \n", "|
| Memorial Spaulding Elementary School | \n", "2070105 | \n", "public | \n", "e | \n", "KG,1,2,3,4,5 | \n", "Newton Centre | \n", "MA | \n", "2459 | \n", "Middlesex County | \n", "42.302044 | \n", "-71.177696 | \n", "304 | \n", "8.0 | \n", "2021.0 | \n", "(42.302044, -71.177696) | \n", "7.062535 | \n", "12.483971 | \n", "|
| Newton South High School | \n", "2070510 | \n", "public | \n", "h | \n", "9,10,11,12 | \n", "Newton Centre | \n", "MA | \n", "2459 | \n", "Middlesex County | \n", "42.314308 | \n", "-71.186493 | \n", "304 | \n", "7.0 | \n", "2021.0 | \n", "(42.314308, -71.186493) | \n", "6.929197 | \n", "11.540487 | \n", "|
| ... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "
| Groton-Dunstable School District | \n", "Florence Roche School | \n", "6730010 | \n", "public | \n", "e | \n", "KG,1,2,3,4 | \n", "Groton | \n", "MA | \n", "1450 | \n", "Middlesex County | \n", "42.616894 | \n", "-71.577682 | \n", "198 | \n", "7.0 | \n", "2021.0 | \n", "(42.616894, -71.577682) | \n", "31.001218 | \n", "19.195452 | \n", "
| Tyngsborough School District | \n", "Tyngsborough Middle School | \n", "3010305 | \n", "public | \n", "m | \n", "6,7,8 | \n", "Tyngsborough | \n", "MA | \n", "1879 | \n", "Middlesex County | \n", "42.696640 | \n", "-71.406586 | \n", "418 | \n", "6.0 | \n", "2021.0 | \n", "(42.69664, -71.406586) | \n", "28.415381 | \n", "17.226867 | \n", "
| Tyngsborough High School | \n", "3010505 | \n", "public | \n", "h | \n", "9,10,11,12 | \n", "Tyngsborough | \n", "MA | \n", "1879 | \n", "Middlesex County | \n", "42.697529 | \n", "-71.408226 | \n", "418 | \n", "8.0 | \n", "2021.0 | \n", "(42.697529, -71.408226) | \n", "28.514503 | \n", "17.319230 | \n", "|
| Groton-Dunstable School District | \n", "Swallow/Union School | \n", "6730005 | \n", "public | \n", "e | \n", "KG,1,2,3,4 | \n", "Dunstable | \n", "MA | \n", "1827 | \n", "Middlesex County | \n", "42.673241 | \n", "-71.482529 | \n", "198 | \n", "9.0 | \n", "2021.0 | \n", "(42.673241, -71.482529) | \n", "29.723989 | \n", "18.012628 | \n", "
| Groton-Dunstable Regional High School | \n", "6730505 | \n", "public | \n", "h | \n", "9,10,11,12,UG | \n", "Groton | \n", "MA | \n", "1450 | \n", "Middlesex County | \n", "42.651093 | \n", "-71.540405 | \n", "198 | \n", "10.0 | \n", "2021.0 | \n", "(42.651093, -71.540405) | \n", "30.877490 | \n", "19.019974 | \n", "
325 rows × 16 columns
\n", "