maq/sn0int
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add geoip information to database

Browse Source
This commit is contained in:
kpcyrd
2018-10-22 18:22:46 +02:00
parent 85fb4dc2be
commit aea38f9db5
10 changed files with 238 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
migrations/2018-10-22-141819_geoip/down.sql Normal file
View File

@@ -0,0 +1,21 @@
PRAGMA foreign_keys=off;
-- ipaddrs
ALTER TABLE ipaddrs RENAME TO _ipaddrs_old;
CREATE TABLE ipaddrs (
id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
family VARCHAR NOT NULL,
value VARCHAR NOT NULL,
unscoped BOOLEAN DEFAULT 0 NOT NULL,
CONSTRAINT ipaddr_unique UNIQUE (value)
);
INSERT INTO ipaddrs (id, family, value, unscoped)
SELECT id, family, value, unscoped
FROM _ipaddrs_old;
DROP TABLE _ipaddrs_old;
PRAGMA foreign_keys=on;

7
migrations/2018-10-22-141819_geoip/up.sql Normal file
View File

@@ -0,0 +1,7 @@
ALTER TABLE ipaddrs ADD COLUMN continent VARCHAR;
ALTER TABLE ipaddrs ADD COLUMN continent_code VARCHAR;
ALTER TABLE ipaddrs ADD COLUMN country VARCHAR;
ALTER TABLE ipaddrs ADD COLUMN country_code VARCHAR;
ALTER TABLE ipaddrs ADD COLUMN city VARCHAR;
ALTER TABLE ipaddrs ADD COLUMN latitude FLOAT;
ALTER TABLE ipaddrs ADD COLUMN longitude FLOAT;

34
modules/dev/geoip.lua
View File

@@ -4,7 +4,35 @@
-- License: GPL-3.0
function run(arg)
x = geoip_lookup(arg['value'])
print('')
print(x)
lookup = geoip_lookup(arg['value'])
if last_err() then return end
fields = {
'continent',
'continent_code',
'country',
'country_code',
'city',
'latitude',
'longitude',
}
update = {}
updated = false
i = 1
while i <= #fields do
f = fields[i]
if lookup[f] ~= arg[f] then
update[f] = lookup[f]
updated = true
end
i = i+1
end
if updated then
db_update('ipaddr', arg, update)
end
end

32
src/db.rs
View File

@@ -61,6 +61,13 @@ impl Database {
Insert::IpAddr(object) => self.insert_ipaddr_struct(&NewIpAddr {
family: &object.family,
value: &object.value,
continent: object.continent.as_ref(),
continent_code: object.continent_code.as_ref(),
country: object.country.as_ref(),
country_code: object.country_code.as_ref(),
city: object.city.as_ref(),
longitude: object.longitude,
latitude: object.latitude,
}),
Insert::SubdomainIpAddr(object) => self.insert_subdomain_ipaddr_struct(&NewSubdomainIpAddr {
subdomain_id: object.subdomain_id,
@@ -144,6 +151,13 @@ impl Database {
let new_ipaddr = NewIpAddr {
family: &family,
value: &ipaddr,
continent: None,
continent_code: None,
country: None,
country_code: None,
city: None,
longitude: None,
latitude: None,
};
self.insert_ipaddr_struct(&new_ipaddr)
@@ -227,27 +241,39 @@ impl Database {
pub fn update_generic(&self, object: &Update) -> Result<i32> {
match object {
Update::Subdomain(object) => self.update_subdomain(object),
Update::IpAddr(object) => self.update_ipaddr(object),
Update::Url(object) => self.update_url(object),
Update::Email(object) => self.update_email(object),
}
}
pub fn update_subdomain(&self, subdomain: &SubdomainUpdate) -> Result<i32> {
diesel::update(subdomains::table)
use schema::subdomains::columns::*;
diesel::update(subdomains::table.filter(id.eq(subdomain.id)))
.set(subdomain)
.execute(&self.db)?;
Ok(subdomain.id)
}
pub fn update_ipaddr(&self, ipaddr: &IpAddrUpdate) -> Result<i32> {
use schema::ipaddrs::columns::*;
diesel::update(ipaddrs::table.filter(id.eq(ipaddr.id)))
.set(ipaddr)
.execute(&self.db)?;
Ok(ipaddr.id)
}
pub fn update_url(&self, url: &UrlUpdate) -> Result<i32> {
diesel::update(urls::table)
use schema::urls::columns::*;
diesel::update(urls::table.filter(id.eq(url.id)))
.set(url)
.execute(&self.db)?;
Ok(url.id)
}
pub fn update_email(&self, email: &EmailUpdate) -> Result<i32> {
diesel::update(emails::table)
use schema::emails::columns::*;
diesel::update(emails::table.filter(id.eq(email.id)))
.set(email)
.execute(&self.db)?;
Ok(email.id)

30
src/geoip.rs
View File

@@ -19,14 +19,18 @@ fn from_geoip_model_names(names: Option<BTreeMap<String, String>>) -> Option<Str
#[derive(Debug, Serialize)]
pub struct Lookup {
continent: Option<Continent>,
country: Option<Country>,
continent: Option<String>,
continent_code: Option<String>,
country: Option<String>,
country_code: Option<String>,
city: Option<String>,
location: Option<Location>,
latitude: Option<f64>,
longitude: Option<f64>,
}
impl From<geoip2::City> for Lookup {
fn from(lookup: geoip2::City) -> Lookup {
// parse maxminddb lookup
let continent = match lookup.continent {
Some(continent) => Continent::from_maxmind(continent),
_ => None,
@@ -44,11 +48,29 @@ impl From<geoip2::City> for Lookup {
_ => None,
};
// flatten datastructure
let (continent, continent_code) = match continent {
Some(x) => (Some(x.name), Some(x.code)),
_ => (None, None),
};
let (country, country_code) = match country {
Some(x) => (Some(x.name), Some(x.code)),
_ => (None, None),
};
let (latitude, longitude) = match location {
Some(x) => (Some(x.latitude), Some(x.longitude)),
_ => (None, None),
};
// return result
Lookup {
continent,
continent_code,
country,
country_code,
city,
location,
latitude,
longitude,
}
}
}

103
src/models/ipaddr.rs
View File

@@ -6,13 +6,20 @@ use std::net;
use std::result;
#[derive(Identifiable, Queryable, Associations, Serialize, PartialEq, Debug)]
#[derive(Identifiable, Queryable, Associations, Serialize, Deserialize, PartialEq, Debug)]
#[table_name="ipaddrs"]
pub struct IpAddr {
pub id: i32,
pub family: String,
pub value: String,
pub unscoped: bool,
pub continent: Option<String>,
pub continent_code: Option<String>,
pub country: Option<String>,
pub country_code: Option<String>,
pub city: Option<String>,
pub latitude: Option<f32>,
pub longitude: Option<f32>,
}
impl fmt::Display for IpAddr {
@@ -135,6 +142,9 @@ pub struct DetailedIpAddr {
value: net::IpAddr,
subdomains: Vec<PrintableSubdomain>,
unscoped: bool,
continent: Option<String>,
country: Option<String>,
city: Option<String>,
}
impl fmt::Display for DetailedIpAddr {
@@ -142,11 +152,40 @@ impl fmt::Display for DetailedIpAddr {
if !self.unscoped {
write!(w, "\x1b[32m#{}\x1b[0m, \x1b[32m{}\x1b[0m", self.id, self.value)?;
if let Some(ref continent) = self.continent {
write!(w, " [{}", continent)?;
if let Some(ref country) = self.country {
write!(w, " / {}", country)?;
}
if let Some(ref city) = self.city {
write!(w, " / {}", city)?;
}
write!(w, "]")?;
}
for subdomain in &self.subdomains {
write!(w, "\n\t\x1b[33m{}\x1b[0m", subdomain)?;
}
} else {
write!(w, "\x1b[90m#{}, {}\x1b[0m", self.id, self.value)?;
write!(w, "\x1b[90m#{}, {}", self.id, self.value)?;
if let Some(ref continent) = self.continent {
write!(w, " [{}", continent)?;
if let Some(ref country) = self.country {
write!(w, " / {}", country)?;
}
if let Some(ref city) = self.city {
write!(w, " / {}", city)?;
}
write!(w, "]")?;
}
write!(w, "\x1b[0m");
for subdomain in &self.subdomains {
write!(w, "\n\t\x1b[90m{}\x1b[0m", subdomain)?;
@@ -170,6 +209,9 @@ impl Detailed for IpAddr {
value: self.value.parse()?,
subdomains,
unscoped: self.unscoped,
continent: self.continent.clone(),
country: self.country.clone(),
city: self.city.clone(),
})
}
}
@@ -179,6 +221,13 @@ impl Detailed for IpAddr {
pub struct NewIpAddr<'a> {
pub family: &'a str,
pub value: &'a str,
pub continent: Option<&'a String>,
pub continent_code: Option<&'a String>,
pub country: Option<&'a String>,
pub country_code: Option<&'a String>,
pub city: Option<&'a String>,
pub latitude: Option<f32>,
pub longitude: Option<f32>,
}
#[derive(Debug, Insertable, Serialize, Deserialize)]
@@ -186,6 +235,56 @@ pub struct NewIpAddr<'a> {
pub struct NewIpAddrOwned {
pub family: String,
pub value: String,
pub continent: Option<String>,
pub continent_code: Option<String>,
pub country: Option<String>,
pub country_code: Option<String>,
pub city: Option<String>,
pub latitude: Option<f32>,
pub longitude: Option<f32>,
}
#[derive(Identifiable, AsChangeset, Serialize, Deserialize, Debug)]
#[table_name="ipaddrs"]
pub struct IpAddrUpdate {
pub id: i32,
pub continent: Option<String>,
pub continent_code: Option<String>,
pub country: Option<String>,
pub country_code: Option<String>,
pub city: Option<String>,
pub latitude: Option<f32>,
pub longitude: Option<f32>,
}
impl fmt::Display for IpAddrUpdate {
fn fmt(&self, w: &mut fmt::Formatter) -> fmt::Result {
let mut updates = Vec::new();
if let Some(ref continent) = self.continent {
updates.push(format!("continent => {:?}", continent));
}
if let Some(ref continent_code) = self.continent_code {
updates.push(format!("continent_code => {:?}", continent_code));
}
if let Some(ref country) = self.country {
updates.push(format!("country => {:?}", country));
}
if let Some(ref country_code) = self.country_code {
updates.push(format!("country_code => {:?}", country_code));
}
if let Some(ref city) = self.city {
updates.push(format!("city => {:?}", city));
}
if let Some(ref latitude) = self.latitude {
updates.push(format!("latitude => {:?}", latitude));
}
if let Some(ref longitude) = self.longitude {
updates.push(format!("longitude => {:?}", longitude));
}
write!(w, "{}", updates.join(", "))
}
}
impl Printable<PrintableIpAddr> for NewIpAddrOwned {

2
src/models/mod.rs
View File

@@ -30,6 +30,7 @@ impl Insert {
#[derive(Debug, Serialize, Deserialize)]
pub enum Update {
Subdomain(SubdomainUpdate),
IpAddr(IpAddrUpdate),
Url(UrlUpdate),
Email(EmailUpdate),
}
@@ -38,6 +39,7 @@ impl fmt::Display for Update {
fn fmt(&self, w: &mut fmt::Formatter) -> fmt::Result {
match self {
Update::Subdomain(update) => write!(w, "{}", update),
Update::IpAddr(update) => write!(w, "{}", update),
Update::Url(update) => write!(w, "{}", update),
Update::Email(update) => write!(w, "{}", update),
}

6
src/models/url.rs
View File

@@ -113,13 +113,13 @@ impl fmt::Display for UrlUpdate {
let mut updates = Vec::new();
if let Some(online) = self.online {
updates.push(format!("online => {:?}, ", online));
updates.push(format!("online => {:?}", online));
}
if let Some(status) = self.status {
updates.push(format!("status => {:?}, ", status));
updates.push(format!("status => {:?}", status));
}
if let Some(ref body) = self.body {
updates.push(format!("body => [{} bytes], ", body.len()));
updates.push(format!("body => [{} bytes]", body.len()));
}
write!(w, "{}", updates.join(", "))

25
src/runtime/db.rs
View File

@@ -85,7 +85,9 @@ pub fn db_update(lua: &mut hlua::Lua, state: Arc<State>) {
Family::Subdomain => structs::from_lua::<Domain>(object)
.map(|x| (x.id, x.to_string()))
.map_err(|e| state.set_error(e)),
Family::IpAddr => bail!("IpAddr doesn't have mutable fields"),
Family::IpAddr => structs::from_lua::<IpAddr>(object)
.map(|x| (x.id, x.to_string()))
.map_err(|e| state.set_error(e)),
Family::SubdomainIpAddr => bail!("Unsupported operation"),
Family::Url => structs::from_lua::<Url>(object)
.map(|x| (x.id, x.to_string()))
@@ -102,20 +104,15 @@ pub fn db_update(lua: &mut hlua::Lua, state: Arc<State>) {
let update = match family {
Family::Domain => bail!("Domain doesn't have mutable fields"),
Family::Subdomain => {
Update::Subdomain(structs::from_lua::<SubdomainUpdate>(update)
.map_err(|e| state.set_error(e))?)
},
Family::IpAddr => bail!("IpAddr doesn't have mutable fields"),
Family::Subdomain => Update::Subdomain(structs::from_lua::<SubdomainUpdate>(update)
.map_err(|e| state.set_error(e))?),
Family::IpAddr => Update::IpAddr(structs::from_lua::<IpAddrUpdate>(update)
.map_err(|e| state.set_error(e))?),
Family::SubdomainIpAddr => bail!("Unsupported operation"),
Family::Url => {
Update::Url(structs::from_lua::<UrlUpdate>(update)
.map_err(|e| state.set_error(e))?)
},
Family::Email => {
Update::Email(structs::from_lua::<EmailUpdate>(update)
.map_err(|e| state.set_error(e))?)
},
Family::Url => Update::Url(structs::from_lua::<UrlUpdate>(update)
.map_err(|e| state.set_error(e))?),
Family::Email => Update::Email(structs::from_lua::<EmailUpdate>(update)
.map_err(|e| state.set_error(e))?),
};
state.db_update(object, update)

7
src/schema.rs
View File

@@ -21,6 +21,13 @@ table! {
family -> Text,
value -> Text,
unscoped -> Bool,
continent -> Nullable<Text>,
continent_code -> Nullable<Text>,
country -> Nullable<Text>,
country_code -> Nullable<Text>,
city -> Nullable<Text>,
latitude -> Nullable<Float>,
longitude -> Nullable<Float>,
}
}