sfjplib

Fork

#!/usr/bin/python
# -*- coding: utf-8 -*-
"""crawlerlib.py"""

from poster.encode import multipart_encode
from poster.streaminghttp import register_openers
import urllib
import urllib2
import cookielib

class CrawlUser(object):
    """Class of user to use crawl"""
    login_url = ""

    def __init__(self):
        self._cookie = None

    def cookie(self):
        return self._cookie

    def set_cookie(self, c):
        self._cookie = c

    def login(self, params):
        c = cookielib.CookieJar()
        p = urllib2.HTTPCookieProcessor(c)
        opener = urllib2.build_opener(p)

        req = opener.open(self.login_url, params)
        self._cookie = c
        return req


class Crawler(object):
    """Class of crawler"""
    def __init__(self, user=None):
        self.set_user(user)

    def set_user(self, user):
        self._user = user

    def _get_cookie(self):
        if self._user and self._user.cookie():
            c = self._user.cookie()
        else:
            c = cookielib.CookieJar()
        return c

    def _set_cookie(self, c):
        self._user.set_cookie(c)

    def get(self, url):
        c = self._get_cookie()
        p = urllib2.HTTPCookieProcessor(c)
        opener = urllib2.build_opener(p)

        res = opener.open(url)
        self._set_cookie(c)
        return res

    def post_form(self, url, params):
        c = self._get_cookie()
        p = urllib2.HTTPCookieProcessor(c)
        opener = urllib2.build_opener(p)

        res = opener.open(url, urllib.urlencode(params))
        self._set_cookie(c)
        return res

    def post_data(self, url, params):
        """
        params = {
            "file_content": open(filename, "rb"),
            "description": "",
            "op": "addFileForStory",
            "sid": sid,
            "Submit": "Submit"
            }
        """
        c = self._get_cookie()
        p = urllib2.HTTPCookieProcessor(c)
        opener = register_openers()
        opener.add_handler(p)

        request = urllib2.Request(url, multipart_encode(params))
        res = opener.open(request)
        self._set_cookie(c)
        return res