#!/usr/bin/env python
# -*- encoding: utf-8 -*-
""" Kaggle ML Project
This script introduces the first use of csv and numpy to manipulate the dataset
of the Kaggle Titanic Project.
* I want first to read the datas,
* then to use them to compute some statistics about the datas,
* and also use them to plot some graphics : on the Plot_*.py files,
--------------------------------------------------------------------------------
About
^^^^^
This script is **still in development**.
The reference page `here on Kaggle <http://www.kaggle.com/c/titanic-gettingStarted>`_.
Documentation
^^^^^^^^^^^^^
The doc is available on-line, on one of my own pages :
* on the cr@ns network `besson/publis/kaggle/
<http://perso.crans.org/besson/publis/kaggle/>`_,
* on the CS department at ENS de Cachan `~lbesson/publis/kaggle/
<http://www.dptinfo.ens-cachan.fr/~lbesson/publis/kaggle/>`_,
Copyrigths
^^^^^^^^^^
(c) Avril-Mai 2013
By Lilian BESSON,
ENS de Cachan (M1 Mathematics & M1 Computer Science MPRI)
mailto:lbesson[AT]ens-cachan.fr
"""
__author__ = 'Lilian BESSON (mailto:lilian.besson[AT]normale.fr)'
__version__ = '1.1.public'
__date__ = 'sam. 20/04/2013 at 22h:59m:51s'
#2#############
# CSV Modules #
import csv as csv #: To read .csv files
import numpy as np #: To compute and use math tools
#3####################
# Scientific modules #
import pylab
################################################################################
# Read the datas (training dataset)
################################################################################
#: Load in the csv file
csv_file_object = csv.reader(open('train.csv', 'rb'))
#: Skip the fist line as it is a header
header = csv_file_object.next()
data = []
for row in csv_file_object:
data.append(row) # adding each row to the data variable
#: Then convert from a list to an array
data = np.array(data)
#: Nom des attributs.
names_columns_train = [ # = header en fait
'survived',
'pclass',
'name',
'sex',
'age',
'sibsp',
'parch',
'ticket',
'fare',
'cabin',
'embarked'
]
# Defines shortcuts
survived = 0 #: ce qu'on veut réussir à prédire !
pclass = 1 #: moyenne + influence, cf Plot_pclass.py
name = 2 #: aucune influence ?
sex = 3 #: grande influence, cf Plot_gender.py
age = 4 #: moyenne influence, cf Plot_age.py
sibsp = 5 #: moyenne + influence, cf Plot_sibsp.py
parch = 6 #: moyenne + influence, cf Plot_parch.py
ticket = 7 #: complexe: numéro du ticket
fare = 8 #: grande influence, cf Plot_fare.py
cabin = 9 #: complexe: localisation dans le bateau
embarked = 10 #: moyenne - influence, cf Plot_port.py
# Now I have an array of 11 columns and 891 rows
# I can access any element I want so the entire first column would
# be data[0::,0].astype(np.flaot) This means all of the columen and column 0
# I have to add the astype command
# as when reading in it thought it was a string so needed to convert
#: Nombre de données d'apprentissage
number_passengers = np.size(data[0::, survived])
#: Nombre de survivants
number_survived = np.sum(data[0::, survived].astype(np.float))
#: Nombre de victimes
number_dead = number_passengers - number_survived
#: Proportion de survivants
proportion_survivors = number_survived / number_passengers
################################################################################
# I can now find the stats of all the women on board
women_only_stats = data[0::, sex] == "female" # This finds where all the women are
men_only_stats = data[0::, sex] == "male" # This finds where all the men are
# I can now find for example the ages of all the women by just placing
# women_only_stats in the '0::' part of the array index. You can test it by
# placing it in the 4 column and it should all read 'female'
women_onboard = data[women_only_stats, survived].astype(np.float) # All women
men_onboard = data[men_only_stats, survived].astype(np.float) # All men !
#: Proportion de femmes
proportion_women = np.size(women_onboard) / float(number_passengers)
#: Proportion de femmes ayant survécues
proportion_women_survived = np.sum(women_onboard) / np.size(women_onboard)
#: Proportion d'hommes
proportion_men = np.size(men_onboard) / float(number_passengers)
#: Proportion d'hommes ayant survécus
proportion_men_survived = np.sum(men_onboard) / np.size(men_onboard)
################################################################################
# I can now find the stats for the three different embarcations points
from_s_onboard = data[ data[0::, embarked] == 'S', survived].astype(np.float)
from_c_onboard = data[ data[0::, embarked] == 'C', survived].astype(np.float)
from_q_onboard = data[ data[0::, embarked] == 'Q', survived].astype(np.float)
#: Proportion de survivants dans les passagers venant de Southampton
proportion_s_survived = np.sum(from_s_onboard) / np.size(from_s_onboard)
#: Proportion de survivants dans les passagers venant de Cherbourg
proportion_c_survived = np.sum(from_c_onboard) / np.size(from_c_onboard)
#: Proportion de survivants dans les passagers venant de Queenstown
proportion_q_survived = np.sum(from_q_onboard) / np.size(from_q_onboard)
################################################################################
# I can now find the stats considering the age of the passengers
known_ages = data[ data[0::, age] != '' , 0::]
#: On ne connais pas l'âge de tous les passagers
proportion_known_ages = np.size(known_ages[0::, survived]) / float(np.size(data[0::,0]))
age_min = known_ages[0::, age].astype(np.float).min() #: Age minimum
age_max = known_ages[0::, age].astype(np.float).max() #: Age max
#: Age moyen, utilisé pour compléter les données
age_mean = known_ages[0::, age].astype(np.float).mean()
known_ages_died = known_ages[ known_ages[0::, survived] == '0', 0:: ]
known_ages_survived = known_ages[ known_ages[0::, survived] == '1', 0:: ]
################################################################################
# I can now find the stats considering the fare of the passengers
known_fares = data[ data[0::, fare] != '' , 0::]
proportion_known_fares = np.size(known_fares[0::, survived]) / float(np.size(data[0::,0]))
fare_min = known_fares[0::, fare].astype(np.float).min() #: Prix min
fare_max = known_fares[0::, fare].astype(np.float).max() #: Prix max
#: Prix moyen, utilisé pour compléter les données
fare_mean = known_fares[0::, fare].astype(np.float).mean()
known_fares_died = known_fares[ known_fares[0::, survived] == '0', 0:: ]
known_fares_survived = known_fares[ known_fares[0::, survived] == '1', 0:: ]
################################################################################
# Read the test dataset
# Now I have my indicator I can read in the test file and write out
# if a women then survived(1) if a man then did not survived (0)
# 1st Read in test
test_file_object = csv.reader(open('test.csv', 'rb'))
header = test_file_object.next()
# Here is tested a completly random guessing:
# RESULT= "Your submission scored 0.49761"
#open_file_object = csv.writer(open("random_guessing.csv", "wb"))
#import random
test_data = []
for row in test_file_object:
test_data.append(row)
# row.insert(0, random.randint(0,1)) # Insert the prediction at the start of the row
# open_file_object.writerow(row) # Write the row to the file
test_data = np.array(test_data)
names_columns_test = [ # = header en fait
# 'survived', # ce qu'on veut réussir à prédire !
'pclass', # moyenne + influence, cf Plot_pclass.py
'name', # aucune influence ?
'sex', # grande influence, cf Plot_gender.py
'age', # moyenne influence, cf Plot_age.py
'sibsp', # moyenne + influence, cf Plot_sibsp.py
'parch', # moyenne + influence, cf Plot_parch.py
'ticket', # complexe: numéro du ticket
'fare', # grande influence, cf Plot_fare.py
'cabin', # complexe: localisation dans le bateau
'embarked' # moyenne - influence, cf Plot_port.py
]
# END :)