import requests
import os 
from PIL import Image
from IPython.display import IFrame
#GET request //# Use single quotation marks for defining string
url='https://www.ibm.com/'  
r=requests.get(url)
#status of the request
r.status_code  
#view request headers //r.request.body
print(r.request.headers)  
#HTTP response header
header=r.headers  
#obtain the date
header['date']  
#obtain the type of data
header['Content-Type']  
r.encoding
#view text
r.text[0:100]  
#write content(image)
path=os.path.join(os.getcwd(),'image.png')
with open(path,'wb') as f:
    f.write(r.content)
Image.open(path)

url_get='http://httpbin.org/get'
#To create a Query string, add a dictionary.
payload={"name":"Joseph","ID":"123"}  
r=requests.get(url_get,params=payload)
r.url  #'http://httpbin.org/get?name=Joseph&ID=123'
#key args in JSON format
r.json()['args']  

url_post='http://httpbin.org/post'
r_post=requests.post(url_post,data=payload)
r_post.url 
r_post.request.body
r_post.json()['form']

from bs4 import BeautifulSoup  # this module helps in web scrapping.
import requests  # this module helps us to download a web page
url = "http://www.ibm.com"
# get the contents of the webpage in text format and store in a variable called data
data  = requests.get(url).text  
# create a soup object using the variable 'data'
soup = BeautifulSoup(data,"html5lib")  
# in html anchor/link is represented by the tag <a>
for link in soup.find_all('a'):  
    print(link.get('href'))
# in html image is represented by the tag <img>
for link in soup.find_all('img'):
    print(link.get('src'))

url = "https://cf-courses-data.s3.us.cloud-object-storage.appdomain.cloud/IBM-DA0321EN-SkillsNetwork/labs/datasets/HTMLColorCodes.html"
# get the contents of the webpage in text format and store in a variable called data
data  = requests.get(url).text  
soup = BeautifulSoup(data,"html5lib")
# in html table is represented by the tag <table>
table = soup.find('table') 
# in html table row is represented by the tag <tr>
for row in table.find_all('tr'): 
    # in html a column is represented by the tag <td>
    cols = row.find_all('td') 
    # store the value in column 3 as color_name
    color_name = cols[2].getText() 
    # store the value in column 4 as color_code
    color_code = cols[3].getText() 
    print("{}--->{}".format(color_name,color_code))

import pandas as pd
dataset_url = "https://cf-courses-data.s3.us.cloud-object-storage.appdomain.cloud/IBM-DA0321EN-SkillsNetwork/LargeData/m1_survey_data.csv"
df = pd.read_csv(dataset_url)

#Display the top & bottom 5 rows and columns from your dataset
df.head()  
df.tail()
#The number of rows in the dataset.
df.shape[0]  
#The number of columns in the dataset.
df.shape[1]  
#Print the datatype of all columns.
df.dtypes  
#Print the mean age of the survey participants.
df["Age"].mean()
#Print how many unique countries are there in the Country column.
df["Country"].nunique()  

import pandas as pd
df = pd.read_csv("https://cf-courses-data.s3.us.cloud-object-storage.appdomain.cloud/IBM-DA0321EN-SkillsNetwork/LargeData/m1_survey_data.csv")

#Find how many duplicate rows exist in the dataframe.
df.duplicated(keep='first').sum()  
#Show duplicated rows
duplicateRows = df[df.duplicated()]  
duplicateRows  
#numbers of duplicate values in the column Respondent
df["Respondent"].duplicated(keep='first').sum()  

#Remove the duplicate rows from the dataframe.
df.drop_duplicates(ignore_index=True, inplace=True)  
#Verify if duplicates were actually dropped.
df.duplicated(keep='first').sum()  
#number of rows and columns left
df.shape  
#numbers of unique rows left in the column Respondent
df["Respondent"].nunique  

#Find the missing values for all columns.
df.isnull().sum()  
#Find out how many rows are missing in the column EdLevel
df["EdLevel"].isnull().sum()  

#Find the value counts for the column WorkLoc.
df["WorkLoc"].value_counts()  
#Impute (replace) all the empty rows in the column WorkLoc with the value that you have identified as majority.
df["WorkLoc"].fillna(value="Office",inplace=True)  
#After imputation there should ideally not be any empty rows in the WorkLoc column.
df["WorkLoc"].isnull().sum()  

#List out the various categories in the column 'CompFreq'
df["CompFreq"].unique()  
#If the CompFreq is Yearly then use the exising value in CompTotal
df["CompFreq"].replace(to_replace="Yearly",value=1,inplace=True)  
#If the CompFreq is Monthly then multiply the value in CompTotal with 12 (months in an year)
df["CompFreq"].replace(to_replace="Monthly",value=12,inplace=True)  
#If the CompFreq is Weekly then multiply the value in CompTotal with 52 (weeks in an year)
df["CompFreq"].replace(to_replace="Weekly",value=52,inplace=True)  
df["CompFreq"].unique()
df["CompFreq"].value_counts()
#it makes comparison of salaries easy.
df['NormalizedAnnualCompensation'] = df["CompTotal"] * df["CompFreq"]  
df["Respondent"].nunique()
df["ConvertedComp"].describe()
df["ConvertedComp"].hist(figsize=(15,4))
df['NormalizedAnnualCompensation'].median()

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
%matplotlib inline
df = pd.read_csv("https://cf-courses-data.s3.us.cloud-object-storage.appdomain.cloud/IBM-DA0321EN-SkillsNetwork/LargeData/m2_survey_data.csv")

#Plot the distribution curve for the column ConvertedComp
plt.figure(figsize=(10,5))
sns.distplot(a=df["ConvertedComp"],bins=20,hist=False)
plt.show()
#Plot the histogram for the column ConvertedComp
plt.figure(figsize=(10,5))
sns.distplot(a=df["ConvertedComp"],bins=20,kde=False)
plt.show()
#the median of the column ConvertedComp
df["ConvertedComp"].median()
#number of responders identified themselves only as a Man
df["Gender"].value_counts()
#the median number of ConvertedComp of responders identified themselves only as a Woman
woman = df[df["Gender"] == "Woman"]
woman["ConvertedComp"].median()
#five number summary for the column Age
df["Age"].describe()

#Find out if outliers exist in the column ConvertedComp using a box plot
plt.figure(figsize=(10,5))
sns.boxplot(x=df.ConvertedComp, data=df)
plt.show()
#Find out the Inter Quartile Range for the column ConvertedComp
df["ConvertedComp"].describe()
#Find out the upper and lower bounds.
Q1 = df["ConvertedComp"].quantile(0.25)
Q3 = df["ConvertedComp"].quantile(0.75)
IQR = Q3 - Q1
print(IQR)
#Identify how many outliers are there in the ConvertedComp column
outliers = (df["ConvertedComp"] < (Q1 - 1.5 * IQR)) | (df["ConvertedComp"] > (Q3 + 1.5 * IQR))
outliers.value_counts()
less = (df["ConvertedComp"] < (Q1 - 1.5 * IQR))
less.value_counts()
more = (df["ConvertedComp"] > (Q3 + 1.5 * IQR))
more.value_counts()
#Create a new dataframe by removing the outliers from the ConvertedComp column
RemoveConvertedcomp = df[~(df["ConvertedComp"] > (Q3 + 1.5 * IQR))]
RemoveConvertedcomp.head()
RemoveConvertedcomp["ConvertedComp"].median()
RemoveConvertedcomp["ConvertedComp"].mean()

df.corr()

!wget https://cf-courses-data.s3.us.cloud-object-storage.appdomain.cloud/IBM-DA0321EN-SkillsNetwork/LargeData/m4_survey_data.sqlite
import sqlite3
import pandas as pd
# open a database connection
conn = sqlite3.connect("m4_survey_data.sqlite") 
# print how many rows are there in the table named 'master'
QUERY = """
SELECT COUNT(*)
FROM master
"""
# the read_sql_query runs the sql query and returns the data as a dataframe
df = pd.read_sql_query(QUERY,conn)
df.head()
# print all the tables names in the database
QUERY = """
SELECT name as Table_Name FROM
sqlite_master WHERE
type = 'table'
"""
# the read_sql_query runs the sql query and returns the data as a dataframe
pd.read_sql_query(QUERY,conn)
#run a group by query
QUERY = """
SELECT Age,COUNT(*) as count
FROM master
group by age
order by age
"""
pd.read_sql_query(QUERY,conn)
#Describe a table
table_name = 'master'  # the table you wish to describe
QUERY = """
SELECT sql FROM sqlite_master
WHERE name= '{}'
""".format(table_name)
df = pd.read_sql_query(QUERY,conn)
print(df.iat[0,0])