Spark 문법

1. 새로운 컬럼추가

.withColumn("변수명", 값)

flights_2 = flights_2.withColumn("duration_hrs", flights_2.air_time/60)
flights_2.show(1)

 

alias로 지정

# avg_speed
avg_speed = (flights_2.distance/(flights_2.air_time/60)).alias("avg_speed")
speed_df = flights.select("origin", "dest", "tailnum", avg_speed)
speed_df.show()

 

 

2. 데이터 필터링

.filter(조건)

result = flights_2.filter("distance >= 1000")
result.show(1)

result2 = flights_2.filter(flights_2.distance > 1000)
result2.show(1)

 

둘 다 같은 결과 나옴

 

다중필터조건

# 다중 필터 조건 걸어보기
filterA = flights_2.origin == "SEA"
filterB = flights_2.dest == "PDX"

tempA = result4.origin == "SEA"

print(type(filterA))
print(type(tempA))

selected = result4.filter(filterA).filter(filterB)
selected.show()

 

3. 변수선택

.select(컬럼명)

# 변수 선택, tailnum, origin, dest
result3 = flights_2.select("tailnum", "origin", "dest")
result3.show(1)

result4 = flights_2.select(flights_2.tailnum, flights_2.origin, flights_2.dest)
result4.show(1)

 

4. Groupby

# origin, PDX 추출, groupby, distance 최소
flights_2.filter(flights_2.origin == "PDX").groupby("tailnum").sum("distance").show()

# carrier
result = (flights_2
          .filter(flights_2.origin == "PDX")
          .filter(flights_2.carrier == "DL")
          .groupby("dest")
          .avg("distance"))
result.show()

 

5. 컬럼명 변경

.withColumnRenamed("기존 컬럼명", "변경될 컬럼명")

# 컬럼명 변경
airports = airports.withColumnRenamed("faa", "dest")
airports.show(1)