{ "id": 3560, "title": "R code to create test set from training", "url": "/competitions/flight/discussion/3560", "author_name": "", "post_date": "2013-01-09T06:55:01.500Z", "votes": 2, "comment_count": 3, "views": 3518, "content": "


\r\ncutoff_days <- read.csv (paste (cv_path, 'days.csv', sep='/'), header=TRUE, stringsAsFactors = FALSE)
\r\ncutoff_days$cutoff_time <- as.POSIXct (cutoff_days$selected_cutoff_time, tz="UTC", format = "%Y-%m-%d %H:%M:%S", origin="1970-1-1" )
\r\n
\r\n#myNms <- c ("flight_history_id", "departure_airport_code", "arrival_airport_code", "published_departure", "published_arrival", "scheduled_gate_departure", "scheduled_gate_arrival", "scheduled_runway_departure", "scheduled_runway_arrival", "actual_gate_arrival",\r\n "actual_runway_arrival", "actual_gate_arrival_mins", "actual_runway_arrival_mins", "airline_icao_code","departure_airport_icao_code", "arrival_airport_icao_code", "creator_code", "scheduled_aircraft_type", "actual_aircraft_type", "scheduled_air_time", "actual_gate_departure",\r\n "actual_runway_departure")
\r\n
\r\nmyNms <- c ("flight_history_id", "departure_airport_code" , "arrival_airport_code", "published_departure" , "published_arrival", "scheduled_gate_departure" , "scheduled_gate_arrival", "scheduled_runway_departure",
\r\n"scheduled_runway_arrival")
\r\n
\r\nfor (myDate in test_dates) {
\r\nprint (paste ("for myDate", myDate, sep=":"))
\r\n
\r\nbase_day_path <- paste (cv_path,myDate, sep="/")
\r\nmyList <- summarize_flight_history_day (base_day_path, test_day_path = NA)
\r\nflighthistory <- myList [[1]]
\r\n
\r\n# remove flights that do not arrive and depart both in the US
\r\nflighthistory_departs_from_us <- flighthistory$departure_airport_icao_code %in% us_icao_codes$icao_code
\r\nflighthistory_arrives_in_us <- flighthistory$arrival_airport_icao_code %in% us_icao_codes$icao_code
\r\nmyRows <- which ((which(flighthistory_arrives_in_us == TRUE) & which(flighthistory_departs_from_us == TRUE)) == TRUE)
\r\nflighthistory <- flighthistory[myRows,]
\r\n
\r\n# remove flights that have an invalid form of datetime - either arrival or departure date
\r\nignoreRows <- c(which (is.na (flighthistory$actual_gate_departure) == TRUE)
\r\n, which (is.na (flighthistory$actual_gate_arrival) == TRUE)
\r\n, which (is.na (flighthistory$actual_runway_arrival) == TRUE)
\r\n, which (is.na (flighthistory$actual_runway_departure) == TRUE))
\r\nignoreRows <- unique (ignoreRows)
\r\n
\r\nflighthistory <- flighthistory [-ignoreRows,]
\r\n
\r\n# only flights that are in air at that time
\r\nflighthistory <- flighthistory [!is.na(flighthistory$actual_gate_arrival),]
\r\nflighthistory <- flighthistory [!is.na(flighthistory$actual_runway_arrival),]
\r\n
\r\n
\r\n# remove flights whose gate_departure time is after runway_departure_time OR whose gate arrival is before runway arrival
\r\nignoreRows <- c(which (flighthistory$actual_gate_departure > flighthistory$actual_runway_departure), which (flighthistory$actual_runway_arrival > flighthistory$actual_gate_arrival))
\r\nignoreRows <- unique (ignoreRows)
\r\n
\r\nflighthistory <- flighthistory [-ignoreRows,]
\r\n
\r\n
\r\n
\r\nflighthistory$cutoff_time <- as.POSIXct (cutoff_days [cutoff_days$folder_name == myDate, "selected_cutoff_time"], tz="UTC", format = "%Y-%m-%d %H:%M:%S", origin="1970-1-1" )
\r\n
\r\ntestflighthistory <- flighthistory [flighthistory$actual_gate_arrival >= flighthistory$cutoff_time,]
\r\n
\r\n
\r\n
\r\nwrite.csv (x=testflighthistory[,myNms] ,file=paste (cv_path, myDate, "test_flights_new.csv", sep= "/"), row.names = FALSE)
\r\n
\r\n

", "messages": [ { "id": "19122", "postDate": "01/09/2013 06:55:01", "content": "


\r\ncutoff_days <- read.csv (paste (cv_path, 'days.csv', sep='/'), header=TRUE, stringsAsFactors = FALSE)
\r\ncutoff_days$cutoff_time <- as.POSIXct (cutoff_days$selected_cutoff_time, tz="UTC", format = "%Y-%m-%d %H:%M:%S", origin="1970-1-1" )
\r\n
\r\n#myNms <- c ("flight_history_id", "departure_airport_code", "arrival_airport_code", "published_departure", "published_arrival", "scheduled_gate_departure", "scheduled_gate_arrival", "scheduled_runway_departure", "scheduled_runway_arrival", "actual_gate_arrival",\r\n "actual_runway_arrival", "actual_gate_arrival_mins", "actual_runway_arrival_mins", "airline_icao_code","departure_airport_icao_code", "arrival_airport_icao_code", "creator_code", "scheduled_aircraft_type", "actual_aircraft_type", "scheduled_air_time", "actual_gate_departure",\r\n "actual_runway_departure")
\r\n
\r\nmyNms <- c ("flight_history_id", "departure_airport_code" , "arrival_airport_code", "published_departure" , "published_arrival", "scheduled_gate_departure" , "scheduled_gate_arrival", "scheduled_runway_departure",
\r\n"scheduled_runway_arrival")
\r\n
\r\nfor (myDate in test_dates) {
\r\nprint (paste ("for myDate", myDate, sep=":"))
\r\n
\r\nbase_day_path <- paste (cv_path,myDate, sep="/")
\r\nmyList <- summarize_flight_history_day (base_day_path, test_day_path = NA)
\r\nflighthistory <- myList [[1]]
\r\n
\r\n# remove flights that do not arrive and depart both in the US
\r\nflighthistory_departs_from_us <- flighthistory$departure_airport_icao_code %in% us_icao_codes$icao_code
\r\nflighthistory_arrives_in_us <- flighthistory$arrival_airport_icao_code %in% us_icao_codes$icao_code
\r\nmyRows <- which ((which(flighthistory_arrives_in_us == TRUE) & which(flighthistory_departs_from_us == TRUE)) == TRUE)
\r\nflighthistory <- flighthistory[myRows,]
\r\n
\r\n# remove flights that have an invalid form of datetime - either arrival or departure date
\r\nignoreRows <- c(which (is.na (flighthistory$actual_gate_departure) == TRUE)
\r\n, which (is.na (flighthistory$actual_gate_arrival) == TRUE)
\r\n, which (is.na (flighthistory$actual_runway_arrival) == TRUE)
\r\n, which (is.na (flighthistory$actual_runway_departure) == TRUE))
\r\nignoreRows <- unique (ignoreRows)
\r\n
\r\nflighthistory <- flighthistory [-ignoreRows,]
\r\n
\r\n# only flights that are in air at that time
\r\nflighthistory <- flighthistory [!is.na(flighthistory$actual_gate_arrival),]
\r\nflighthistory <- flighthistory [!is.na(flighthistory$actual_runway_arrival),]
\r\n
\r\n
\r\n# remove flights whose gate_departure time is after runway_departure_time OR whose gate arrival is before runway arrival
\r\nignoreRows <- c(which (flighthistory$actual_gate_departure > flighthistory$actual_runway_departure), which (flighthistory$actual_runway_arrival > flighthistory$actual_gate_arrival))
\r\nignoreRows <- unique (ignoreRows)
\r\n
\r\nflighthistory <- flighthistory [-ignoreRows,]
\r\n
\r\n
\r\n
\r\nflighthistory$cutoff_time <- as.POSIXct (cutoff_days [cutoff_days$folder_name == myDate, "selected_cutoff_time"], tz="UTC", format = "%Y-%m-%d %H:%M:%S", origin="1970-1-1" )
\r\n
\r\ntestflighthistory <- flighthistory [flighthistory$actual_gate_arrival >= flighthistory$cutoff_time,]
\r\n
\r\n
\r\n
\r\nwrite.csv (x=testflighthistory[,myNms] ,file=paste (cv_path, myDate, "test_flights_new.csv", sep= "/"), row.names = FALSE)
\r\n
\r\n

", "rawMarkdown": "", "votes": null }, { "id": "19141", "postDate": "01/10/2013 03:01:42", "content": "

Hi Black Magic,

\r\n

I wanted to point out a helpful date package: lubridate. It's a tool created by Hadley Wickham, the author of plyr, ggplot2, and stringr. It's pretty handy for working with dates.

\r\n

Also, I see you are doing a check of ICAO against some list. I'm pretty sure that all continential US ICAOs beging with K. I don't think it makes much difference computationally, but the rules are pretty clear about disallowing outside data sources.

", "rawMarkdown": "", "votes": null }, { "id": "19143", "postDate": "01/10/2013 03:19:10", "content": "

Hi Ricardo,

\r\n

[quote=ricardo sachez;19141] the rules are pretty clear about disallowing outside data sources.[/quote]

\r\n

A list of ICAO_code has been provided (see Reference/usairporticaocodes).

\r\n

Best,

", "rawMarkdown": "", "votes": null }, { "id": "19145", "postDate": "01/10/2013 04:27:15", "content": "

yes I am using same ICAO codes.
\r\nProblem is that traditional Cross validation is not working here. I have lined up some good algos but I need CV to work to optimize

", "rawMarkdown": "", "votes": null } ], "comments": [ { "id": 19141, "author_name": "rickyars", "author_url": "", "post_date": "01/10/2013 03:01:42", "content": "

Hi Black Magic,

\r\n

I wanted to point out a helpful date package: lubridate. It's a tool created by Hadley Wickham, the author of plyr, ggplot2, and stringr. It's pretty handy for working with dates.

\r\n

Also, I see you are doing a check of ICAO against some list. I'm pretty sure that all continential US ICAOs beging with K. I don't think it makes much difference computationally, but the rules are pretty clear about disallowing outside data sources.

", "votes": null, "replies": [] }, { "id": 19143, "author_name": "jay670790", "author_url": "", "post_date": "01/10/2013 03:19:10", "content": "

Hi Ricardo,

\r\n

[quote=ricardo sachez;19141] the rules are pretty clear about disallowing outside data sources.[/quote]

\r\n

A list of ICAO_code has been provided (see Reference/usairporticaocodes).

\r\n

Best,

", "votes": null, "replies": [] }, { "id": 19145, "author_name": "rkirana", "author_url": "", "post_date": "01/10/2013 04:27:15", "content": "

yes I am using same ICAO codes.
\r\nProblem is that traditional Cross validation is not working here. I have lined up some good algos but I need CV to work to optimize

", "votes": null, "replies": [] } ], "raw_markdown_by_id": { "19122": "", "19141": "", "19143": "", "19145": "" }, "source": "meta" }