Browse Source

Parametrized script to insert data, and moved reading code out

master
Maxime Wack 6 years ago
parent
commit
348d372a37
2 changed files with 24 additions and 16 deletions
  1. +7
    -16
      geoip.R
  2. +17
    -0
      viz.R

+ 7
- 16
geoip.R View File

@@ -1,11 +1,15 @@
library(tidyverse)
library(RSQLite)
library(ipapi)
library(leaflet)

dbConnect(SQLite(), "access.db") -> db
commandArgs(trailingOnly = T) -> cmdargs
dbfile <- cmdargs[1]
logfile <- cmdargs[2]

read_delim("access.log", delim = " ", col_names = c("ip", "null1", "user", "timestamp", "zone", "req", "status", "size", "referer", "agent", "null2")) %>%
dbConnect(SQLite(), dbfile) -> db

logfile %>%
read_delim(delim = " ", col_names = c("ip", "null1", "user", "timestamp", "zone", "req", "status", "size", "referer", "agent", "null2")) %>%
mutate_all(na_if, "-") %>%
filter(user %>% is.na) %>%
filter(!ip %in% c("192.168.0.254", "164.2.255.244")) %>%
@@ -37,16 +41,3 @@ if (length(ips) > 0)
select(ip = query, city, country, lat, lon) %>%
dbWriteTable(conn = db, name = "geoip", value = ., append = T)
}

db %>%
tbl("access") %>%
left_join(db %>% tbl("geoip")) %>%
collect -> geoaccess

geoaccess %>%
filter(status != 404,
!agent %>% str_detect("bot")) %>%
distinct(ip, lon, lat) %>%
leaflet %>%
addProviderTiles(providers$CartoDB.Positron) %>%
addMarkers(~lon, ~lat, clusterOptions = markerClusterOptions())

+ 17
- 0
viz.R View File

@@ -0,0 +1,17 @@
library(tidyverse)
library(leaflet)

src_sqlite("access.db") %>%
tbl("access") %>%
left_join(db %>% tbl("geoip")) %>%
collect %>%
mutate(timestamp = timestamp %>% as.POSIXct(origin = "1970-01-01")) %>%
mutate_at(vars(city, country, agent), factor) -> geoaccess

geoaccess %>%
filter(status != 404,
!agent %>% str_detect("bot")) %>%
distinct(ip, lon, lat) %>%
leaflet %>%
addProviderTiles(providers$CartoDB.Positron) %>%
addMarkers(~lon, ~lat, clusterOptions = markerClusterOptions())

Loading…
Cancel
Save