working on it ...

Filters

Explore Public Snippets

Sort by

Found 50 snippets matching: urllib2

    public by lbottaro  4407  1  7  0

    How to fetch Internet Resources Using urllib2 in python

    urllib2 is a Python module for fetching URLs (Uniform Resource Locators). It offers a simple interface, based on the urlopen function. This method is capable of fetching URLs using a variety of different protocols. In this example you can see a specific exception handlers for HTTPError and URLError.
    import os, urllib2
    
    def getURLResult():
        try:      
            intRes = urllib2.urlopen("http://your_URL")
        except urllib2.HTTPError, e:
            print "HTTPError Error: " + str(e.code) 
        except urllib2.URLError, e:
            print "URLError Error: " + str(e.reason) 
        print "ok"

    public by lbottaro  9700  25  8  0

    How to parse the JSON result of a jenkins job in python

    This snippet shows how to get and parse the JSON format of the last execution of a Jenkins job. You have to specify: Jenkins_server - The Jenkins server name/IP address http_port - The port used by Jenkins server job_name - The job name
    import urllib2
    import json 
    
    jenkins_job_dir="http://<Jenkins_server>:<http_port>/view/All/job/<job_name>/lastBuild/api/json"
    def getBuildResult():
        try:      
            jenkinsStream    = urllib2.urlopen(jenkins_job_dir)
    
        except urllib2.HTTPError, e:
            print "URL Error: " + str(e.code) 
            return "FAILURE"
        
        try:
            buildStatusJson = json.load( jenkinsStream )
        except:
            print "Failed to parse json"
            return "FAILURE"
    
        if buildStatusJson.has_key( "result" ):      
            print "build status job " + buildStatusJson["result"]
            if buildStatusJson["result"] != "SUCCESS" :
                return "FAILURE"
        else:
            return "FAILURE"
        return "SUCCESS"
    
    print "loading"
    getBuildResult()
    print "Done"

    external by osya  91  0  1  0

    Парсер избиркома на Python urllib, urllib2 & lxml

    Парсер избиркома на Python urllib, urllib2 & lxml: gistfile1.py
    # -*- coding: utf-8 -*-
    from django.core.management.base import AppCommand
    from commission.izbirkom_parser import parser
    from commission.models import Comission
    import lxml.html as html
    import urllib2, urllib
    import json
    import re
    import time
    
    
    class parser:
        site = 'http://www.moscow_city.vybory.izbirkom.ru/'
        tree_url = 'moscow_city/ik_tree/?first=1&id2=%s'
        page_url = 'moscow_city/ik/%s'
        def get_page(self, page_url):
            #page = html.parse()
            response = urllib2.urlopen('%s/%s' % (self.site,page_url))
            text = response.read()
            response.close()
            return text.decode('cp1251')
        def read_tree(self, id=''):
            json_tree = json.loads(self.get_page(self.tree_url % id))
            ids = []
            elements = json_tree[0]['children']
            for element in elements:
                if id == '':
                    ids.append(element['id'])
                else:
    
                    if element['id'] == id:
                        for uik in element['children']:
                            ids.append(uik['id'])
            return ids
    
        def parse_page(self, text, pid = False , type='gik'):
            page = html.document_fromstring(text)
            name = page.cssselect('div.center-colm h2')[0].text_content()
            #c = Comission.objects.get(name=name)
            #if c:
            #    print name + "- NO"
            #    return c
    
            address = page.cssselect('div.center-colm p b span')[0].attrib['rel']
            p = re.compile('\d+')
            phone = ''.join(p.findall(page.cssselect('div.center-colm p')[2].text_content()))
            info = html.tostring(page.cssselect('div.table')[0])
            #print 'http://geocode-maps.yandex.ru/1.x/?geocode='+address+'&type=json'
    
            yandex = urllib2.urlopen('http://geocode-maps.yandex.ru/1.x/?'+urllib.urlencode({'geocode':address.encode('utf-8'),'type':'json'}))
            yandex_page = yandex.read()
            yandex.close()
            ya = html.document_fromstring(yandex_page)
            geo = ya.cssselect('pos')[0].text_content()
    
            comission = Comission(type=type, name=name, address=address, phone=phone, info=info, geo=geo)
            if pid:
                comission.parent_comission = pid
            c = comission.save()
            print name + "- OK"
            time.sleep(1)
            return comission
    
        def __init__(self):
            pid = self.parse_page(self.get_page(self.page_url % '2772000301927'))
            tik_ids = self.read_tree()
            for tik_id in tik_ids:
                id = self.parse_page(self.get_page(self.page_url % tik_id), pid, 'tik')
                uik_ids = self.read_tree(tik_id)
                for uik_id in uik_ids:
                    self.parse_page(self.get_page(self.page_url % uik_id), id, 'uik')
    
    
    class Command( AppCommand ):
        help = u'Автодобавление ТИК/УИК'
        def handle(self, *args, **options):
            parser()
    
    

    external by Eutychus Towett  57  0  1  0

    Make a post request using python urllib2 library by populating the payload dictionary in make_request function with the POST body and updating the url

    Make a post request using python urllib2 library by populating the payload dictionary in make_request function with the POST body and updating the url: client.py
    #!/usr/bin/python2.7
    
    """Client.py
    
    Make a POST request
    """
    
    import urllib
    import urllib2
    
    url = ""
    
    
    def make_request():
        payload = {
            'param': 'value'
        }
        return urllib2.urlopen(url, urllib.urlencode(payload)).read()
    
    if __name__ == '__main__':
        print make_request()
    
    
    
    

    external by curious-eyes  1720  31  4  0

    urllib2 sample sending OPTIONS method.

    urllib2 sample sending OPTIONS method.: urllib2_options.py
    #!/usr/bin/env python
    # -*- coding: utf-8 -*-
    import urllib2
    
    opener = urllib2.build_opener(urllib2.HTTPHandler)
    request = urllib2.Request('http://example.com', data='your_put_data')
    request.add_header('Content-Type', 'your/contenttype')
    request.get_method = lambda: 'OPTIONS'
    url = opener.open(request)
    
    
    

    external by Averroes  45  0  1  0

    urllib2 urlopen iterator

    urllib2 urlopen iterator: urllib2_urlopen_iterator.py
    #!/usr/bin/env python
    # encoding: utf-8
    #
    # Copyright (c) 2008 Doug Hellmann All rights reserved.
    #
    """Simple example with urllib.urlopen().
    """
    #end_pymotw_header
    
    import urllib2
    
    response = urllib2.urlopen('http://localhost:8080/')
    for line in response:
        print line.rstrip()
    
    
    

    external by Emory-M  521  33  3  0

    Download m3u8 and merge ts file with python urllib2

    Download m3u8 and merge ts file with python urllib2: m3u8dl.py
    # coding: utf8
    
    import sys, urllib2
    import socks
    from sockshandler import SocksiPyHandler
    from urlparse import urlparse
    
    if len(sys.argv) < 2:
    	print 'Usage: python %s [url]' % sys.argv[0]
    	exit()
    
    def parseurl():
    	url = sys.argv[1]
    	urlgroup = urlparse(url)
    	host = urlgroup.scheme + '://' + urlgroup.hostname
    	file = urlgroup.path.split('/')[-1].split('.')[0] + '.ts'
    	opener = urllib2.build_opener(SocksiPyHandler(socks.SOCKS5, '127.0.0.1'))
    	response = opener.open(url)
    	print 'm3u8 done!'
    	count = 0
    	while response.readline():
    		line = response.readline().strip()
    		if not line.startswith('#') and line != '':
    			ts = opener.open(host+line).read()
    			tsfile = open(file, 'ab')
    			tsfile.write(ts)
    			count += 1
    			print 'part %s done!' % count
    	print 'file: ' + file
    
    if __name__ == "__main__":
    	try:
    		parseurl()
    	except KeyboardInterrupt:
    		print 'exit'
    
    

    external by Averroes  34  0  1  0

    urllib2 upload files

    urllib2 upload files: urllib2_upload_files.py
    #!/usr/bin/env python
    # encoding: utf-8
    #
    # Copyright (c) 2008 Doug Hellmann All rights reserved.
    #
    """
    """
    #end_pymotw_header
    
    import itertools
    import mimetools
    import mimetypes
    from cStringIO import StringIO
    import urllib
    import urllib2
    
    class MultiPartForm(object):
        """Accumulate the data to be used when posting a form."""
    
        def __init__(self):
            self.form_fields = []
            self.files = []
            self.boundary = mimetools.choose_boundary()
            return
        
        def get_content_type(self):
            return 'multipart/form-data; boundary=%s' % self.boundary
    
        def add_field(self, name, value):
            """Add a simple field to the form data."""
            self.form_fields.append((name, value))
            return
    
        def add_file(self, fieldname, filename, fileHandle,
                     mimetype=None):
            """Add a file to be uploaded."""
            body = fileHandle.read()
            if mimetype is None:
                mimetype = ( mimetypes.guess_type(filename)[0]
                             or
                             'application/octet-stream'
                             )
            self.files.append((fieldname, filename, mimetype, body))
            return
        
        def __str__(self):
            """Return a string representing the form data,
            including attached files.
            """
            # Build a list of lists, each containing "lines" of the
            # request.  Each part is separated by a boundary string.
            # Once the list is built, return a string where each
            # line is separated by '\r\n'.  
            parts = []
            part_boundary = '--' + self.boundary
            
            # Add the form fields
            parts.extend(
                [ part_boundary,
                  'Content-Disposition: form-data; name="%s"' % name,
                  '',
                  value,
                ]
                for name, value in self.form_fields
                )
            
            # Add the files to upload
            parts.extend([
                part_boundary,
                'Content-Disposition: file; name="%s"; filename="%s"' % \
                   (field_name, filename),
                'Content-Type: %s' % content_type,
                '',
                body,
              ]
              for field_name, filename, content_type, body in self.files
              )
            
            # Flatten the list and add closing boundary marker, and
            # then return CR+LF separated data
            flattened = list(itertools.chain(*parts))
            flattened.append('--' + self.boundary + '--')
            flattened.append('')
            return '\r\n'.join(flattened)
    
    if __name__ == '__main__':
        # Create the form with simple fields
        form = MultiPartForm()
        form.add_field('firstname', 'Doug')
        form.add_field('lastname', 'Hellmann')
        
        # Add a fake file
        form.add_file(
            'biography', 'bio.txt', 
            fileHandle=StringIO('Python developer and blogger.'))
    
        # Build the request
        request = urllib2.Request('http://localhost:8080/')
        request.add_header(
            'User-agent',
            'PyMOTW (http://www.doughellmann.com/PyMOTW/)')
        body = str(form)
        request.add_header('Content-type', form.get_content_type())
        request.add_header('Content-length', len(body))
        request.add_data(body)
    
        print
        print 'OUTGOING DATA:'
        print request.get_data()
    
        print
        print 'SERVER RESPONSE:'
        print urllib2.urlopen(request).read()
    
    
    

    external by Averroes  22  0  1  0

    urllib2 http get args

    urllib2 http get args: urllib2_http_get_args.py
    #!/usr/bin/env python
    # encoding: utf-8
    #
    # Copyright (c) 2008 Doug Hellmann All rights reserved.
    #
    """
    """
    #end_pymotw_header
    
    import urllib
    import urllib2
    
    query_args = { 'q':'query string', 'foo':'bar' }
    encoded_args = urllib.urlencode(query_args)
    print 'Encoded:', encoded_args
    
    url = 'http://localhost:8080/?' + encoded_args
    print urllib2.urlopen(url).read()
    
    
    

    external by Github  424  9  3  0

    ntlm authorization python urllib2 mechanize

    ntlm authorization python urllib2 mechanize : ntlm-authorization
    import urllib2
    from ntlm import HTTPNtlmAuthHandler
    
    user = '<myusername>'
    password = "<mypass>"
    url = "https://somesite.com"
    
    passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
    passman.add_password(None, url, user, password)
    # create the NTLM authentication handler
    auth_NTLM = HTTPNtlmAuthHandler.HTTPNtlmAuthHandler(passman)
    
    import mechanize
    browser = mechanize.Browser()
    handlersToKeep = []
    
    for handler in browser.handlers:
        if not isinstance(handler,
        (mechanize._http.HTTPRobotRulesProcessor)):
            handlersToKeep.append(handler)
    
    browser.handlers = handlersToKeep
    browser.add_handler(auth_NTLM)
    
    response = browser.open(url)
    print(response.read())
    
    
    • Public Snippets
    • Channels Snippets