[weboob] [PATCH 1/1] wetboobs: added module ilmatieteenlaitos

Matthieu Weber mweber+weboob at free.fr
Wed Feb 4 23:08:53 CET 2015


From: Matthieu Weber <mweber at free.fr>


Signed-off-by: Matthieu Weber <mweber+weboob at free.fr>
---
 modules/ilmatieteenlaitos/__init__.py |   23 +++++++
 modules/ilmatieteenlaitos/browser.py  |   40 +++++++++++
 modules/ilmatieteenlaitos/favicon.png |  Bin 0 -> 2213 bytes
 modules/ilmatieteenlaitos/module.py   |   48 +++++++++++++
 modules/ilmatieteenlaitos/pages.py    |  122 +++++++++++++++++++++++++++++++++
 modules/ilmatieteenlaitos/test.py     |   36 ++++++++++
 weboob/browser/elements.py            |    4 +-
 weboob/capabilities/weather.py        |   10 +--
 8 files changed, 276 insertions(+), 7 deletions(-)
 create mode 100644 modules/ilmatieteenlaitos/__init__.py
 create mode 100644 modules/ilmatieteenlaitos/browser.py
 create mode 100644 modules/ilmatieteenlaitos/favicon.png
 create mode 100644 modules/ilmatieteenlaitos/module.py
 create mode 100644 modules/ilmatieteenlaitos/pages.py
 create mode 100644 modules/ilmatieteenlaitos/test.py

diff --git a/modules/ilmatieteenlaitos/__init__.py b/modules/ilmatieteenlaitos/__init__.py
new file mode 100644
index 0000000..427fe8e
--- /dev/null
+++ b/modules/ilmatieteenlaitos/__init__.py
@@ -0,0 +1,23 @@
+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2015 Matthieu Weber
+#
+# This file is part of weboob.
+#
+# weboob is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# weboob is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with weboob. If not, see <http://www.gnu.org/licenses/>.
+
+
+from .module import IlmatieteenlaitosModule
+
+__all__ = ['IlmatieteenlaitosModule']
diff --git a/modules/ilmatieteenlaitos/browser.py b/modules/ilmatieteenlaitos/browser.py
new file mode 100644
index 0000000..7d73952
--- /dev/null
+++ b/modules/ilmatieteenlaitos/browser.py
@@ -0,0 +1,40 @@
+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2015 Matthieu Weber
+#
+# This file is part of weboob.
+#
+# weboob is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# weboob is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with weboob. If not, see <http://www.gnu.org/licenses/>.
+
+from weboob.browser.browsers import PagesBrowser
+from weboob.browser.url import URL
+from .pages import WeatherPage, SearchCitiesPage
+
+__all__ = ['IlmatieteenlaitosBrowser']
+
+
+class IlmatieteenlaitosBrowser(PagesBrowser):
+    BASEURL = 'http://ilmatieteenlaitos.fi'
+    cities = URL('/etusivu\?p_p_id=locationmenuportlet_WAR_fmiwwwweatherportlets&p_p_lifecycle=2&p_p_state=normal&p_p_mode=view&p_p_cacheability=cacheLevelFull&term=(?P<pattern>.*)', SearchCitiesPage)
+    weather_query = URL('/paikallissaa\?p_p_id=locationmenuportlet_WAR_fmiwwwweatherportlets&p_p_lifecycle=1&p_p_state=normal&p_p_mode=view&_locationmenuportlet_WAR_fmiwwwweatherportlets_action=changelocation')
+    weather = URL('/saa/(?P<city_url>.*)', WeatherPage)
+
+    def iter_city_search(self, pattern):
+        return self.cities.go(pattern=pattern).iter_cities()
+
+    def iter_forecast(self, city):
+        return self.weather_query.go(data={"place": city.name, "forecast": "short"}).iter_forecast()
+
+    def get_current(self, city):
+        return self.weather_query.go(data={"place": city.name, "forecast": "short"}).get_current()
diff --git a/modules/ilmatieteenlaitos/favicon.png b/modules/ilmatieteenlaitos/favicon.png
new file mode 100644
index 0000000000000000000000000000000000000000..1265787208d58cce4b0b878856a93f76e503f8e9
GIT binary patch
literal 2213
zcmV;W2wL}vP)<h;3K|Lk000e1NJLTq002M$002M;0ssI2B at 5<>00009a7bBm000XU
z000XU0RWnu7ytkO2XskIMF-yk1QjeBbEgBe0000PbVXQnLvL+uWo~o;Lvm$dbY)~9
zcWHEJAV*0}P*;Ht7XSbVA4x<(RA}DqT7PU)*A at Q!18_`Y8!!xebs}P5wi_>oBr6K0
zSvS&do}6s*2#U0_RYXV)X*J0SN;Hd&DQyW^)TxA2q-1NWDO+KdFb2j#G-fre=uFm^
z8tfG6AcxK#;53e%9}czi_Q#8y5FE#TFE(1Yqx_TS_s;#kbI*P6-g7<{Ns?F&i$<e9
zpU>rT5d=XHgv;f+bLUPV5C9+&iDWXFQmNEvG#ZUYsZ>fN5)Oy6EaQS?W`RJ!VzKD;
z`c%i&>-83kB at hTO1i+#<10+e>?e^y8W~<doT at eTbTCKLIs7R$!<>%*1rP4KP)?{U6
z0hpSa8XX<=dc8wKLr$l&zrVk)uP+!3#+aB)rjn8p7K at dM8JL-w>FMbai^bIAx#}I?
z`_&nn&6cVko6S~VU!RwkNBI$p#XUVeGcz-(`bmudK at f7eoEoxpU(@cBH^r}yh+iMs
z-ZlE3Gn}lK==7B9_{g8c)Zf{&XLULq<x?(~69kc#6bgsKCX=bHt&M`t-Swh#eD%V#
z-m>!5Z}T^<W+z(ffg59&pT*3B1=p`j%q=Y~7cX7}U^E(=o11w&UNXRvA}}~Os8*{Z
zkw~t3hqAV-KfH0VJH>o<+jsdo`O3JzPP^ZpxYm$hpdRvgV`HPo<Kc3-4u?abP%Hz1
zbLY-gR#pPoU4Qas-j63EOGo9fmj6=m!|@{{6L#bD?e3)kty<S`Ufb2x1;AplR8&+j
zB5>lwiMqNv0J=9m`bTtg^6!S)BHoc*{8{kP>5#1|a{t5RAAj$QU)R>w0%&h<KX~w9
zf;$uCYinz3U0t1Ez4SYsLn)yL(9s`mx-tm>GY>*nDpNuaVDH+)mo8lr2n2O?b*-(f
zbY_5x{nzqU@}|o-$9Z(#(B*%V3D~)C{qDox()-AMw&_AeMaA8_cN5G*d;(P73)V{u
zUbWLiU-C~V2V?(zdie-#4&~+L!C-K)^}vq%&Y at Oq@hg2a(dP}%N6*AI-~Per)29Jc
zR#pxU4lYd~91g41YHHPfG9jchCMSyGj~86~M)@ykb0`V?q@$w)fLg5%hr>${Fquq|
zNMv{Y$y8VEoNfCAt0x}4%!I?<wTJib-yexYOeRyJH83zRpwVb@)jM$DLNf3BI3^6D
z^88{y(q0MF_hH-5!yb=^Ac(@k!Z<TPlBB_40HCbxqKO{Blg<P`GMNN&dgSQQqW}yB
z!~CGn5$Nshb-UfA`<fE$6qc}@acmw!+U!}@OFEs-?RNL}_AX2zk|Z~5*x>j3pMGoj
z(Op`yww*wMbIH4yB+_+>xvZ>AEEeCpcaJ(h%+7$_Zuk5BrTd!bqX&?G8)>!zpFyY7
z`Tc&o-TqJtH8(c{;O~5azV9cz;7=uUBa at W?;J|?cF~G9~#>U31R%@<$2Qj&h0fEnv
z5!)56$;->LTCHPaW3vQoHX8ty!N|z5^xw#o?b)*@2Ad7w)TvVdCcpj^eU~{QWRE9*
zJHsrZiXaqhb~GBbTCIZhQbx;HJc0~bfzKci2&`6XG#X|5d_DjL-~TBimumyatZB7c
z06w3O?Q*#Qh!i^+`4a>)OrWT!2!PAwViN=bAi8D~BY*LBZ)r7^N(F!*h*<(5u85Jp
ztZ;@2<mczl5};COEV_!3zZGAm`owPS%Zy&yQYn=}_^-BDh4cURn2gKia>Zh?-|x>o
zK0^Nx1K{~Tf|YWap5Xy)?T<bci9~FfOa>rl1+zX)l#}7uPZak8kjZ3hrBVq%$eCv3
z at 5>dbJ|UYLWip`G>jj`xD%l#120%`P*<N5O<?>N)-y2K at 92yz|pwVb%32;W49i}E$
zXE=qNPUkEEDun`rA29M46l9n{e}DgM3P~gq0N4NW4kLeGh%!u|udfe)L?U5xI2^rR
zAH45XWi!~2`PXF<{dI{s7!2z5dJc!f22fK|1AzapkLbG$<zPHJV+5$)E(MznpscJ6
zfYaQ@$g4Mnau}0O7=SZp&ctA|0SJXcy<R`;xT at hZI2nHaL}tSOKJS#r<I(H&LZNW3
zE<{y*CqMW-0|HV^W at UgtXJ=;&FuppaMHY+I-Q5i!^8TCjgL7+hy2Ibn(gL8nyE`VV
znClOaB)MhF7Ps5|+?zM(7pP0GCN5Aj34afHyt=wtE|=fBbt^_-uDs4-vCL*O09VI;
zK9}wwH&khgJ!{pv#>PefX0v&|uH~T?Q(<AD(P$iYT;1yVGtF_v1aK<T5&nkrT93zL
zG#cl(XYgp7H5?A-<m5ymkyKmrbH<6{hv(xKT4)oX8uwf-Hxvpj5P>Xg^6_{)hr<Ej
z;>%BcbM+X_!_BpA%ZT1C`U539a5x-~N?79bi4_Wk#bN<)?HAgu*|Z8io?N&T%Yy#1
zO&6-FssLClmbl`T_*#jIii-C3_TYW5`?YeK9PHcS+sJU?Qc{CpFxcMSzL;z!ky at p-
zwe`@ULxT0v;#d0UG`)G at 8V%1_!dZD`Z)BvOM&?viRRx2=W5<p)G%TtzN|)RQKusa<
zwP*N9v)m>vf+swz7Xlui;EA&2C1;eyQEUN7kK~M<OgI$^MJN=azHxtj at yV?p|7Nk{
zo0!jjN1q)RJ%Cm6p9!D)JJ>7Z?$Vxr_gmG~)zsH26bdDSKG_+F*}%Yn!C-K^-2h7W
zHSu at 8Fu$>xjA{`8W*&rZc1{i)iS6$@J3A at 8qFgRFo6QTpwM*Et2w$ImYd8hn=&}l5
n|M#TFUlH+loE^|(lHdOa(4MKQ4lv*=00000NkvXXu0mjfXX7h}

literal 0
HcmV?d00001

diff --git a/modules/ilmatieteenlaitos/module.py b/modules/ilmatieteenlaitos/module.py
new file mode 100644
index 0000000..2ea1b7e
--- /dev/null
+++ b/modules/ilmatieteenlaitos/module.py
@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2015 Matthieu Weber
+#
+# This file is part of weboob.
+#
+# weboob is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# weboob is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with weboob. If not, see <http://www.gnu.org/licenses/>.
+
+
+from weboob.capabilities.weather import CapWeather, CityNotFound
+from weboob.tools.backend import Module
+from weboob.capabilities.base import find_object
+from .browser import IlmatieteenlaitosBrowser
+
+
+__all__ = ['IlmatieteenlaitosModule']
+
+class IlmatieteenlaitosModule(Module, CapWeather):
+    NAME = 'ilmatieteenlaitos'
+    MAINTAINER = u'Matthieu Weber'
+    EMAIL = 'mweber+weboob at free.fr'
+    VERSION = '1.1'
+    DESCRIPTION = 'Get forecasts from the Ilmatieteenlaitos.fi website'
+    LICENSE = 'AGPLv3+'
+    BROWSER = IlmatieteenlaitosBrowser
+
+    def get_current(self, city_id):
+        return self.browser.get_current(self.get_city(city_id))
+
+    def iter_forecast(self, city_id):
+        return self.browser.iter_forecast(self.get_city(city_id))
+
+    def iter_city_search(self, pattern):
+        return self.browser.iter_city_search(pattern)
+
+    def get_city(self, _id):
+        return find_object(self.iter_city_search(_id), id=_id, error=CityNotFound)
diff --git a/modules/ilmatieteenlaitos/pages.py b/modules/ilmatieteenlaitos/pages.py
new file mode 100644
index 0000000..1c4410d
--- /dev/null
+++ b/modules/ilmatieteenlaitos/pages.py
@@ -0,0 +1,122 @@
+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2015 Matthieu Weber
+#
+# This file is part of weboob.
+#
+# weboob is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# weboob is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with weboob. If not, see <http://www.gnu.org/licenses/>.
+
+from datetime import date
+from itertools import imap, ifilter
+
+from weboob.browser.pages import JsonPage, HTMLPage
+from weboob.browser.elements import ItemElement, ListElement, method, SkipItem
+from weboob.capabilities.weather import Forecast, Current, City, Temperature
+from weboob.browser.filters.json import Dict
+from weboob.browser.filters.html import CleanHTML
+from weboob.browser.filters.standard import Filter, CleanText, CleanDecimal, Regexp, Format, Date, Lower
+
+
+class DictElement(ListElement):
+    def find_elements(self):
+        if self.item_xpath is not None:
+            for el in self.el:
+                yield el
+        else:
+            yield self.el
+
+class Id(Filter):
+    def filter(self, txt):
+        return txt.split(", ")[0]
+
+class SearchCitiesPage(JsonPage):
+    @method
+    class iter_cities(DictElement):
+        item_xpath = '.'
+        ignore_duplicate = True
+
+        class item(ItemElement):
+            klass = City
+
+            obj_id = Id(Dict('id'))
+            obj_name = Dict('value')
+
+class WeatherPage(HTMLPage):
+    @method
+    class iter_forecast(ListElement):
+        item_xpath = '//div[contains(@class, "mid") and contains(@class, "local-weather-forecast")]//tr[@class="meteogram-dates"]/td'
+
+        class item(ItemElement):
+            klass = Forecast
+
+            obj_id = CleanText('.//span/@title')
+            def obj_date(self):
+                months = [u'tammikuuta', u'helmikuuta', u'maaliskuuta', u'huhtikuuta', u'toukokuuta', u'kesäkuuta',
+                        u'heinäkuuta', u'elokuuta', u'syyskuuta', u'lokakuuta', u'marraskuuta', u'joulukuuta']
+                d = CleanText('.//span/@title')(self).split()
+                return date(int(d[2]), months.index(d[1])+1, int(d[0].strip(".")))
+
+
+            def temperatures(self):
+                offset = int(CleanText('string(sum(./preceding-sibling::td/@colspan))')(self))
+                length = int(CleanText('@colspan')(self))
+                temps = CleanText('../../../tbody/tr[@class="meteogram-temperatures"]/td[position() > %d and position() <= %d]/span' % (offset, offset+length))(self)
+                return [float(_.strip(u'\xb0')) for _ in temps.split()]
+
+            def obj_low(self):
+                return Temperature(min(self.temperatures()), u'C')
+
+            def obj_high(self):
+                return Temperature(max(self.temperatures()), u'C')
+
+            def obj_text(self):
+                offset = int(CleanText('string(sum(./preceding-sibling::td/@colspan))')(self))
+                length = int(CleanText('@colspan')(self))
+                hour_test = '../../tr[@class="meteogram-times"]/td[position() > %d and position() <= %d and .//text() = "%%s"]' % (offset, offset+length)
+                hour_offset = 'string(count(%s/preceding-sibling::td)+1)' % (hour_test)
+                values = [
+                        '../../../tbody/tr[@class="meteogram-weather-symbols"]/td[position() = %d]/div/@title',
+                        '../../../tbody/tr[@class="meteogram-apparent-temperatures"]/td[position() = %d]/div/@title',
+                        '../../../tbody/tr[@class="meteogram-wind-symbols"]/td[position() = %d]/div/@title',
+                        '../../../tbody/tr[@class="meteogram-probabilities-of-precipitation"]/td[position() = %d]/div/@title',
+                        '../../../tbody/tr[@class="meteogram-hourly-precipitation-values"]/td[position() = %d]/span/@title',
+                    ]
+                def descriptive_text_for_hour(hour):
+                    hour_exists = CleanText(hour_test % hour)(self) == hour
+                    if hour_exists:
+                        offset = int(CleanText(hour_offset % hour)(self))
+                        def info_for_value(value):
+                            return CleanText(value % offset)(self).replace(u'edeltävän tunnin ', u'')
+                        return ("klo %s: " % hour) + ", ".join(ifilter(bool, imap(info_for_value, values)))
+
+                return "\n" + "\n".join(ifilter(bool, imap(descriptive_text_for_hour, ["02", "14"])))
+
+
+    @method
+    class get_current(ItemElement):
+        klass = Current
+
+        obj_id = date.today()
+        obj_date = Date(Regexp(CleanText('//table[@class="observation-text"]//span[@class="time-stamp"]'), r'^(\d+\.\d+.\d+)'))
+        obj_text = Format(u'%s, %s, %s',
+                CleanText(u'(//table[@class="observation-text"])//tr[2]/td[2]'),
+                CleanText(u'(//table[@class="observation-text"])//tr[5]/td[1]'),
+                CleanText(u'(//table[@class="observation-text"])//tr[4]/td[2]'),
+                )
+
+        def obj_temp(self):
+            path = u'//table[@class="observation-text"]//span[@class="parameter-name" and text() = "Lämpötila"]/../span[@class="parameter-value"]'
+            temp = CleanDecimal(Regexp(CleanText(path), r'^([^ \xa0]+)'), replace_dots=True)(self)
+            unit = Regexp(CleanText(path), r'\xb0(\w)')(self)
+            return Temperature(float(temp), unit)
diff --git a/modules/ilmatieteenlaitos/test.py b/modules/ilmatieteenlaitos/test.py
new file mode 100644
index 0000000..1fc1025
--- /dev/null
+++ b/modules/ilmatieteenlaitos/test.py
@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2015 Matthieu Weber
+#
+# This file is part of weboob.
+#
+# weboob is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# weboob is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with weboob. If not, see <http://www.gnu.org/licenses/>.
+
+
+from weboob.tools.test import BackendTest
+
+
+class IlmatieteenlaitosTest(BackendTest):
+    MODULE = 'ilmatieteenlaitos'
+
+    def test_ilmatieteenlaitos(self):
+        l = list(self.backend.iter_city_search('helsinki'))
+        self.assertTrue(len(l) > 0)
+
+        city = l[0]
+        current = self.backend.get_current(city.id)
+        self.assertTrue(current.temp.value > -50 and current.temp.value < 50)
+
+        forecasts = list(self.backend.iter_forecast(city.id))
+        self.assertTrue(len(forecasts) > 0)
diff --git a/weboob/browser/elements.py b/weboob/browser/elements.py
index 2ce2e8d..be06ff5 100644
--- a/weboob/browser/elements.py
+++ b/weboob/browser/elements.py
@@ -271,8 +271,8 @@ class ItemElement(AbstractElement):
             # Help debugging as tracebacks do not give us the key
             self.logger.warning('Attribute %s raises %s' % (key, repr(e)))
             raise
-        logger = getLogger('b2filters')
-        logger.log(DEBUG_FILTERS, "%s.%s = %r" % (self._random_id, key, value))
+        #logger = getLogger('b2filters')
+        #logger.log(DEBUG_FILTERS, "%s.%s = %r" % (self._random_id, key, value))
         setattr(self.obj, key, value)
 
 
diff --git a/weboob/capabilities/weather.py b/weboob/capabilities/weather.py
index 888cdb9..31f0c35 100644
--- a/weboob/capabilities/weather.py
+++ b/weboob/capabilities/weather.py
@@ -43,17 +43,17 @@ class Temperature(BaseObject):
         if not self.unit:
             return u'%s' % int(round(self.value))
         elif self.unit == 'F':
-            return u'%s°F' % int(round(self.value))
+            return u'%s °F' % int(round(self.value))
         else:
-            return u'%s°F' % int(round((self.value * 9.0 / 5.0) + 32))
+            return u'%s °F' % int(round((self.value * 9.0 / 5.0) + 32))
 
     def ascelsius(self):
         if not self.unit:
             return u'%s' % int(round(self.value))
         elif self.unit == 'C':
-            return u'%s°C' % int(round(self.value))
+            return u'%s °C' % int(round(self.value))
         else:
-            return u'%s°C' % int(round((self.value - 32.0) * 5.0 / 9.0))
+            return u'%s °C' % int(round((self.value - 32.0) * 5.0 / 9.0))
 
     def __repr__(self):
         if self.value is not None and self.unit:
@@ -87,7 +87,7 @@ class Current(BaseObject):
 
     def __init__(self, date='', temp=None, text=None, unit=None):
         BaseObject.__init__(self, unicode(date))
-        self.date = date
+        self.date = date or None
         self.text = text
         self.temp = Temperature(temp, unit)
 
-- 
1.7.10.4



More information about the weboob mailing list