From 87165e91f27336d93f98b2e79a1dd82199b69548 Mon Sep 17 00:00:00 2001 From: panni Date: Sun, 28 Apr 2019 06:02:12 +0200 Subject: [PATCH 01/23] core: update to subliminal_patch:head --- libs/certifi/__init__.py | 4 +- libs/certifi/cacert.pem | 388 ++++++++++++++++++ libs/certifi/core.py | 22 - libs/cfscrape/__init__.py | 151 +++++-- libs/cfscrape/browsers.json | 80 ++++ libs/cfscrape/browsers_br.json | 336 +++++++++++++++ libs/requests/__version__.py | 4 +- libs/requests/models.py | 2 +- libs/requests/sessions.py | 13 +- libs/requests/utils.py | 4 +- libs/subliminal_patch/core.py | 7 +- libs/subliminal_patch/http.py | 140 ++++--- .../providers/opensubtitles.py | 7 +- libs/subliminal_patch/providers/subscene.py | 6 +- libs/subliminal_patch/providers/titlovi.py | 50 +-- libs/subliminal_patch/subtitle.py | 14 +- libs/subzero/constants.py | 2 +- libs/urllib3/__init__.py | 2 +- libs/urllib3/contrib/pyopenssl.py | 3 + libs/urllib3/poolmanager.py | 7 +- libs/urllib3/response.py | 8 +- libs/urllib3/util/retry.py | 3 +- libs/urllib3/util/ssl_.py | 7 +- 23 files changed, 1076 insertions(+), 184 deletions(-) create mode 100644 libs/cfscrape/browsers.json create mode 100644 libs/cfscrape/browsers_br.json diff --git a/libs/certifi/__init__.py b/libs/certifi/__init__.py index 50f2e1301..632db8e13 100644 --- a/libs/certifi/__init__.py +++ b/libs/certifi/__init__.py @@ -1,3 +1,3 @@ -from .core import where, old_where +from .core import where -__version__ = "2018.10.15" +__version__ = "2019.03.09" diff --git a/libs/certifi/cacert.pem b/libs/certifi/cacert.pem index e75d85b38..84636dde7 100644 --- a/libs/certifi/cacert.pem +++ b/libs/certifi/cacert.pem @@ -4268,3 +4268,391 @@ rYy0UGYwEAYJKwYBBAGCNxUBBAMCAQAwCgYIKoZIzj0EAwMDaAAwZQIwJsdpW9zV 57LnyAyMjMPdeYwbY9XJUpROTYJKcx6ygISpJcBMWm1JKWB4E+J+SOtkAjEA2zQg Mgj/mkkCtojeFK9dbJlxjRo/i9fgojaGHAeCOnZT/cKi7e97sIBPWA9LUzm9 -----END CERTIFICATE----- + +# Issuer: CN=GTS Root R1 O=Google Trust Services LLC +# Subject: CN=GTS Root R1 O=Google Trust Services LLC +# Label: "GTS Root R1" +# Serial: 146587175971765017618439757810265552097 +# MD5 Fingerprint: 82:1a:ef:d4:d2:4a:f2:9f:e2:3d:97:06:14:70:72:85 +# SHA1 Fingerprint: e1:c9:50:e6:ef:22:f8:4c:56:45:72:8b:92:20:60:d7:d5:a7:a3:e8 +# SHA256 Fingerprint: 2a:57:54:71:e3:13:40:bc:21:58:1c:bd:2c:f1:3e:15:84:63:20:3e:ce:94:bc:f9:d3:cc:19:6b:f0:9a:54:72 +-----BEGIN CERTIFICATE----- +MIIFWjCCA0KgAwIBAgIQbkepxUtHDA3sM9CJuRz04TANBgkqhkiG9w0BAQwFADBH +MQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExM +QzEUMBIGA1UEAxMLR1RTIFJvb3QgUjEwHhcNMTYwNjIyMDAwMDAwWhcNMzYwNjIy +MDAwMDAwWjBHMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNl +cnZpY2VzIExMQzEUMBIGA1UEAxMLR1RTIFJvb3QgUjEwggIiMA0GCSqGSIb3DQEB +AQUAA4ICDwAwggIKAoICAQC2EQKLHuOhd5s73L+UPreVp0A8of2C+X0yBoJx9vaM +f/vo27xqLpeXo4xL+Sv2sfnOhB2x+cWX3u+58qPpvBKJXqeqUqv4IyfLpLGcY9vX +mX7wCl7raKb0xlpHDU0QM+NOsROjyBhsS+z8CZDfnWQpJSMHobTSPS5g4M/SCYe7 +zUjwTcLCeoiKu7rPWRnWr4+wB7CeMfGCwcDfLqZtbBkOtdh+JhpFAz2weaSUKK0P +fyblqAj+lug8aJRT7oM6iCsVlgmy4HqMLnXWnOunVmSPlk9orj2XwoSPwLxAwAtc +vfaHszVsrBhQf4TgTM2S0yDpM7xSma8ytSmzJSq0SPly4cpk9+aCEI3oncKKiPo4 +Zor8Y/kB+Xj9e1x3+naH+uzfsQ55lVe0vSbv1gHR6xYKu44LtcXFilWr06zqkUsp +zBmkMiVOKvFlRNACzqrOSbTqn3yDsEB750Orp2yjj32JgfpMpf/VjsPOS+C12LOO +Rc92wO1AK/1TD7Cn1TsNsYqiA94xrcx36m97PtbfkSIS5r762DL8EGMUUXLeXdYW +k70paDPvOmbsB4om3xPXV2V4J95eSRQAogB/mqghtqmxlbCluQ0WEdrHbEg8QOB+ +DVrNVjzRlwW5y0vtOUucxD/SVRNuJLDWcfr0wbrM7Rv1/oFB2ACYPTrIrnqYNxgF +lQIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNV +HQ4EFgQU5K8rJnEaK0gnhS9SZizv8IkTcT4wDQYJKoZIhvcNAQEMBQADggIBADiW +Cu49tJYeX++dnAsznyvgyv3SjgofQXSlfKqE1OXyHuY3UjKcC9FhHb8owbZEKTV1 +d5iyfNm9dKyKaOOpMQkpAWBz40d8U6iQSifvS9efk+eCNs6aaAyC58/UEBZvXw6Z +XPYfcX3v73svfuo21pdwCxXu11xWajOl40k4DLh9+42FpLFZXvRq4d2h9mREruZR +gyFmxhE+885H7pwoHyXa/6xmld01D1zvICxi/ZG6qcz8WpyTgYMpl0p8WnK0OdC3 +d8t5/Wk6kjftbjhlRn7pYL15iJdfOBL07q9bgsiG1eGZbYwE8na6SfZu6W0eX6Dv +J4J2QPim01hcDyxC2kLGe4g0x8HYRZvBPsVhHdljUEn2NIVq4BjFbkerQUIpm/Zg +DdIx02OYI5NaAIFItO/Nis3Jz5nu2Z6qNuFoS3FJFDYoOj0dzpqPJeaAcWErtXvM ++SUWgeExX6GjfhaknBZqlxi9dnKlC54dNuYvoS++cJEPqOba+MSSQGwlfnuzCdyy +F62ARPBopY+Udf90WuioAnwMCeKpSwughQtiue+hMZL77/ZRBIls6Kl0obsXs7X9 +SQ98POyDGCBDTtWTurQ0sR8WNh8M5mQ5Fkzc4P4dyKliPUDqysU0ArSuiYgzNdws +E3PYJ/HQcu51OyLemGhmW/HGY0dVHLqlCFF1pkgl +-----END CERTIFICATE----- + +# Issuer: CN=GTS Root R2 O=Google Trust Services LLC +# Subject: CN=GTS Root R2 O=Google Trust Services LLC +# Label: "GTS Root R2" +# Serial: 146587176055767053814479386953112547951 +# MD5 Fingerprint: 44:ed:9a:0e:a4:09:3b:00:f2:ae:4c:a3:c6:61:b0:8b +# SHA1 Fingerprint: d2:73:96:2a:2a:5e:39:9f:73:3f:e1:c7:1e:64:3f:03:38:34:fc:4d +# SHA256 Fingerprint: c4:5d:7b:b0:8e:6d:67:e6:2e:42:35:11:0b:56:4e:5f:78:fd:92:ef:05:8c:84:0a:ea:4e:64:55:d7:58:5c:60 +-----BEGIN CERTIFICATE----- +MIIFWjCCA0KgAwIBAgIQbkepxlqz5yDFMJo/aFLybzANBgkqhkiG9w0BAQwFADBH +MQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExM +QzEUMBIGA1UEAxMLR1RTIFJvb3QgUjIwHhcNMTYwNjIyMDAwMDAwWhcNMzYwNjIy +MDAwMDAwWjBHMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNl +cnZpY2VzIExMQzEUMBIGA1UEAxMLR1RTIFJvb3QgUjIwggIiMA0GCSqGSIb3DQEB +AQUAA4ICDwAwggIKAoICAQDO3v2m++zsFDQ8BwZabFn3GTXd98GdVarTzTukk3Lv +CvptnfbwhYBboUhSnznFt+4orO/LdmgUud+tAWyZH8QiHZ/+cnfgLFuv5AS/T3Kg +GjSY6Dlo7JUle3ah5mm5hRm9iYz+re026nO8/4Piy33B0s5Ks40FnotJk9/BW9Bu +XvAuMC6C/Pq8tBcKSOWIm8Wba96wyrQD8Nr0kLhlZPdcTK3ofmZemde4wj7I0BOd +re7kRXuJVfeKH2JShBKzwkCX44ofR5GmdFrS+LFjKBC4swm4VndAoiaYecb+3yXu +PuWgf9RhD1FLPD+M2uFwdNjCaKH5wQzpoeJ/u1U8dgbuak7MkogwTZq9TwtImoS1 +mKPV+3PBV2HdKFZ1E66HjucMUQkQdYhMvI35ezzUIkgfKtzra7tEscszcTJGr61K +8YzodDqs5xoic4DSMPclQsciOzsSrZYuxsN2B6ogtzVJV+mSSeh2FnIxZyuWfoqj +x5RWIr9qS34BIbIjMt/kmkRtWVtd9QCgHJvGeJeNkP+byKq0rxFROV7Z+2et1VsR +nTKaG73VululycslaVNVJ1zgyjbLiGH7HrfQy+4W+9OmTN6SpdTi3/UGVN4unUu0 +kzCqgc7dGtxRcw1PcOnlthYhGXmy5okLdWTK1au8CcEYof/UVKGFPP0UJAOyh9Ok +twIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNV +HQ4EFgQUu//KjiOfT5nK2+JopqUVJxce2Q4wDQYJKoZIhvcNAQEMBQADggIBALZp +8KZ3/p7uC4Gt4cCpx/k1HUCCq+YEtN/L9x0Pg/B+E02NjO7jMyLDOfxA325BS0JT +vhaI8dI4XsRomRyYUpOM52jtG2pzegVATX9lO9ZY8c6DR2Dj/5epnGB3GFW1fgiT +z9D2PGcDFWEJ+YF59exTpJ/JjwGLc8R3dtyDovUMSRqodt6Sm2T4syzFJ9MHwAiA +pJiS4wGWAqoC7o87xdFtCjMwc3i5T1QWvwsHoaRc5svJXISPD+AVdyx+Jn7axEvb +pxZ3B7DNdehyQtaVhJ2Gg/LkkM0JR9SLA3DaWsYDQvTtN6LwG1BUSw7YhN4ZKJmB +R64JGz9I0cNv4rBgF/XuIwKl2gBbbZCr7qLpGzvpx0QnRY5rn/WkhLx3+WuXrD5R +RaIRpsyF7gpo8j5QOHokYh4XIDdtak23CZvJ/KRY9bb7nE4Yu5UC56GtmwfuNmsk +0jmGwZODUNKBRqhfYlcsu2xkiAhu7xNUX90txGdj08+JN7+dIPT7eoOboB6BAFDC +5AwiWVIQ7UNWhwD4FFKnHYuTjKJNRn8nxnGbJN7k2oaLDX5rIMHAnuFl2GqjpuiF +izoHCBy69Y9Vmhh1fuXsgWbRIXOhNUQLgD1bnF5vKheW0YMjiGZt5obicDIvUiLn +yOd/xCxgXS/Dr55FBcOEArf9LAhST4Ldo/DUhgkC +-----END CERTIFICATE----- + +# Issuer: CN=GTS Root R3 O=Google Trust Services LLC +# Subject: CN=GTS Root R3 O=Google Trust Services LLC +# Label: "GTS Root R3" +# Serial: 146587176140553309517047991083707763997 +# MD5 Fingerprint: 1a:79:5b:6b:04:52:9c:5d:c7:74:33:1b:25:9a:f9:25 +# SHA1 Fingerprint: 30:d4:24:6f:07:ff:db:91:89:8a:0b:e9:49:66:11:eb:8c:5e:46:e5 +# SHA256 Fingerprint: 15:d5:b8:77:46:19:ea:7d:54:ce:1c:a6:d0:b0:c4:03:e0:37:a9:17:f1:31:e8:a0:4e:1e:6b:7a:71:ba:bc:e5 +-----BEGIN CERTIFICATE----- +MIICDDCCAZGgAwIBAgIQbkepx2ypcyRAiQ8DVd2NHTAKBggqhkjOPQQDAzBHMQsw +CQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzEU +MBIGA1UEAxMLR1RTIFJvb3QgUjMwHhcNMTYwNjIyMDAwMDAwWhcNMzYwNjIyMDAw +MDAwWjBHMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZp +Y2VzIExMQzEUMBIGA1UEAxMLR1RTIFJvb3QgUjMwdjAQBgcqhkjOPQIBBgUrgQQA +IgNiAAQfTzOHMymKoYTey8chWEGJ6ladK0uFxh1MJ7x/JlFyb+Kf1qPKzEUURout +736GjOyxfi//qXGdGIRFBEFVbivqJn+7kAHjSxm65FSWRQmx1WyRRK2EE46ajA2A +DDL24CejQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1Ud +DgQWBBTB8Sa6oC2uhYHP0/EqEr24Cmf9vDAKBggqhkjOPQQDAwNpADBmAjEAgFuk +fCPAlaUs3L6JbyO5o91lAFJekazInXJ0glMLfalAvWhgxeG4VDvBNhcl2MG9AjEA +njWSdIUlUfUk7GRSJFClH9voy8l27OyCbvWFGFPouOOaKaqW04MjyaR7YbPMAuhd +-----END CERTIFICATE----- + +# Issuer: CN=GTS Root R4 O=Google Trust Services LLC +# Subject: CN=GTS Root R4 O=Google Trust Services LLC +# Label: "GTS Root R4" +# Serial: 146587176229350439916519468929765261721 +# MD5 Fingerprint: 5d:b6:6a:c4:60:17:24:6a:1a:99:a8:4b:ee:5e:b4:26 +# SHA1 Fingerprint: 2a:1d:60:27:d9:4a:b1:0a:1c:4d:91:5c:cd:33:a0:cb:3e:2d:54:cb +# SHA256 Fingerprint: 71:cc:a5:39:1f:9e:79:4b:04:80:25:30:b3:63:e1:21:da:8a:30:43:bb:26:66:2f:ea:4d:ca:7f:c9:51:a4:bd +-----BEGIN CERTIFICATE----- +MIICCjCCAZGgAwIBAgIQbkepyIuUtui7OyrYorLBmTAKBggqhkjOPQQDAzBHMQsw +CQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzEU +MBIGA1UEAxMLR1RTIFJvb3QgUjQwHhcNMTYwNjIyMDAwMDAwWhcNMzYwNjIyMDAw +MDAwWjBHMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZp +Y2VzIExMQzEUMBIGA1UEAxMLR1RTIFJvb3QgUjQwdjAQBgcqhkjOPQIBBgUrgQQA +IgNiAATzdHOnaItgrkO4NcWBMHtLSZ37wWHO5t5GvWvVYRg1rkDdc/eJkTBa6zzu +hXyiQHY7qca4R9gq55KRanPpsXI5nymfopjTX15YhmUPoYRlBtHci8nHc8iMai/l +xKvRHYqjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1Ud +DgQWBBSATNbrdP9JNqPV2Py1PsVq8JQdjDAKBggqhkjOPQQDAwNnADBkAjBqUFJ0 +CMRw3J5QdCHojXohw0+WbhXRIjVhLfoIN+4Zba3bssx9BzT1YBkstTTZbyACMANx +sbqjYAuG7ZoIapVon+Kz4ZNkfF6Tpt95LY2F45TPI11xzPKwTdb+mciUqXWi4w== +-----END CERTIFICATE----- + +# Issuer: CN=UCA Global G2 Root O=UniTrust +# Subject: CN=UCA Global G2 Root O=UniTrust +# Label: "UCA Global G2 Root" +# Serial: 124779693093741543919145257850076631279 +# MD5 Fingerprint: 80:fe:f0:c4:4a:f0:5c:62:32:9f:1c:ba:78:a9:50:f8 +# SHA1 Fingerprint: 28:f9:78:16:19:7a:ff:18:25:18:aa:44:fe:c1:a0:ce:5c:b6:4c:8a +# SHA256 Fingerprint: 9b:ea:11:c9:76:fe:01:47:64:c1:be:56:a6:f9:14:b5:a5:60:31:7a:bd:99:88:39:33:82:e5:16:1a:a0:49:3c +-----BEGIN CERTIFICATE----- +MIIFRjCCAy6gAwIBAgIQXd+x2lqj7V2+WmUgZQOQ7zANBgkqhkiG9w0BAQsFADA9 +MQswCQYDVQQGEwJDTjERMA8GA1UECgwIVW5pVHJ1c3QxGzAZBgNVBAMMElVDQSBH +bG9iYWwgRzIgUm9vdDAeFw0xNjAzMTEwMDAwMDBaFw00MDEyMzEwMDAwMDBaMD0x +CzAJBgNVBAYTAkNOMREwDwYDVQQKDAhVbmlUcnVzdDEbMBkGA1UEAwwSVUNBIEds +b2JhbCBHMiBSb290MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAxeYr +b3zvJgUno4Ek2m/LAfmZmqkywiKHYUGRO8vDaBsGxUypK8FnFyIdK+35KYmToni9 +kmugow2ifsqTs6bRjDXVdfkX9s9FxeV67HeToI8jrg4aA3++1NDtLnurRiNb/yzm +VHqUwCoV8MmNsHo7JOHXaOIxPAYzRrZUEaalLyJUKlgNAQLx+hVRZ2zA+te2G3/R +VogvGjqNO7uCEeBHANBSh6v7hn4PJGtAnTRnvI3HLYZveT6OqTwXS3+wmeOwcWDc +C/Vkw85DvG1xudLeJ1uK6NjGruFZfc8oLTW4lVYa8bJYS7cSN8h8s+1LgOGN+jIj +tm+3SJUIsUROhYw6AlQgL9+/V087OpAh18EmNVQg7Mc/R+zvWr9LesGtOxdQXGLY +D0tK3Cv6brxzks3sx1DoQZbXqX5t2Okdj4q1uViSukqSKwxW/YDrCPBeKW4bHAyv +j5OJrdu9o54hyokZ7N+1wxrrFv54NkzWbtA+FxyQF2smuvt6L78RHBgOLXMDj6Dl +NaBa4kx1HXHhOThTeEDMg5PXCp6dW4+K5OXgSORIskfNTip1KnvyIvbJvgmRlld6 +iIis7nCs+dwp4wwcOxJORNanTrAmyPPZGpeRaOrvjUYG0lZFWJo8DA+DuAUlwznP +O6Q0ibd5Ei9Hxeepl2n8pndntd978XplFeRhVmUCAwEAAaNCMEAwDgYDVR0PAQH/ +BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFIHEjMz15DD/pQwIX4wV +ZyF0Ad/fMA0GCSqGSIb3DQEBCwUAA4ICAQATZSL1jiutROTL/7lo5sOASD0Ee/oj +L3rtNtqyzm325p7lX1iPyzcyochltq44PTUbPrw7tgTQvPlJ9Zv3hcU2tsu8+Mg5 +1eRfB70VVJd0ysrtT7q6ZHafgbiERUlMjW+i67HM0cOU2kTC5uLqGOiiHycFutfl +1qnN3e92mI0ADs0b+gO3joBYDic/UvuUospeZcnWhNq5NXHzJsBPd+aBJ9J3O5oU +b3n09tDh05S60FdRvScFDcH9yBIw7m+NESsIndTUv4BFFJqIRNow6rSn4+7vW4LV +PtateJLbXDzz2K36uGt/xDYotgIVilQsnLAXc47QN6MUPJiVAAwpBVueSUmxX8fj +y88nZY41F7dXyDDZQVu5FLbowg+UMaeUmMxq67XhJ/UQqAHojhJi6IjMtX9Gl8Cb +EGY4GjZGXyJoPd/JxhMnq1MGrKI8hgZlb7F+sSlEmqO6SWkoaY/X5V+tBIZkbxqg +DMUIYs6Ao9Dz7GjevjPHF1t/gMRMTLGmhIrDO7gJzRSBuhjjVFc2/tsvfEehOjPI ++Vg7RE+xygKJBJYoaMVLuCaJu9YzL1DV/pqJuhgyklTGW+Cd+V7lDSKb9triyCGy +YiGqhkCyLmTTX8jjfhFnRR8F/uOi77Oos/N9j/gMHyIfLXC0uAE0djAA5SN4p1bX +UB+K+wb1whnw0A== +-----END CERTIFICATE----- + +# Issuer: CN=UCA Extended Validation Root O=UniTrust +# Subject: CN=UCA Extended Validation Root O=UniTrust +# Label: "UCA Extended Validation Root" +# Serial: 106100277556486529736699587978573607008 +# MD5 Fingerprint: a1:f3:5f:43:c6:34:9b:da:bf:8c:7e:05:53:ad:96:e2 +# SHA1 Fingerprint: a3:a1:b0:6f:24:61:23:4a:e3:36:a5:c2:37:fc:a6:ff:dd:f0:d7:3a +# SHA256 Fingerprint: d4:3a:f9:b3:54:73:75:5c:96:84:fc:06:d7:d8:cb:70:ee:5c:28:e7:73:fb:29:4e:b4:1e:e7:17:22:92:4d:24 +-----BEGIN CERTIFICATE----- +MIIFWjCCA0KgAwIBAgIQT9Irj/VkyDOeTzRYZiNwYDANBgkqhkiG9w0BAQsFADBH +MQswCQYDVQQGEwJDTjERMA8GA1UECgwIVW5pVHJ1c3QxJTAjBgNVBAMMHFVDQSBF +eHRlbmRlZCBWYWxpZGF0aW9uIFJvb3QwHhcNMTUwMzEzMDAwMDAwWhcNMzgxMjMx +MDAwMDAwWjBHMQswCQYDVQQGEwJDTjERMA8GA1UECgwIVW5pVHJ1c3QxJTAjBgNV +BAMMHFVDQSBFeHRlbmRlZCBWYWxpZGF0aW9uIFJvb3QwggIiMA0GCSqGSIb3DQEB +AQUAA4ICDwAwggIKAoICAQCpCQcoEwKwmeBkqh5DFnpzsZGgdT6o+uM4AHrsiWog +D4vFsJszA1qGxliG1cGFu0/GnEBNyr7uaZa4rYEwmnySBesFK5pI0Lh2PpbIILvS +sPGP2KxFRv+qZ2C0d35qHzwaUnoEPQc8hQ2E0B92CvdqFN9y4zR8V05WAT558aop +O2z6+I9tTcg1367r3CTueUWnhbYFiN6IXSV8l2RnCdm/WhUFhvMJHuxYMjMR83dk +sHYf5BA1FxvyDrFspCqjc/wJHx4yGVMR59mzLC52LqGj3n5qiAno8geK+LLNEOfi +c0CTuwjRP+H8C5SzJe98ptfRr5//lpr1kXuYC3fUfugH0mK1lTnj8/FtDw5lhIpj +VMWAtuCeS31HJqcBCF3RiJ7XwzJE+oJKCmhUfzhTA8ykADNkUVkLo4KRel7sFsLz +KuZi2irbWWIQJUoqgQtHB0MGcIfS+pMRKXpITeuUx3BNr2fVUbGAIAEBtHoIppB/ +TuDvB0GHr2qlXov7z1CymlSvw4m6WC31MJixNnI5fkkE/SmnTHnkBVfblLkWU41G +sx2VYVdWf6/wFlthWG82UBEL2KwrlRYaDh8IzTY0ZRBiZtWAXxQgXy0MoHgKaNYs +1+lvK9JKBZP8nm9rZ/+I8U6laUpSNwXqxhaN0sSZ0YIrO7o1dfdRUVjzyAfd5LQD +fwIDAQABo0IwQDAdBgNVHQ4EFgQU2XQ65DA9DfcS3H5aBZ8eNJr34RQwDwYDVR0T +AQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAYYwDQYJKoZIhvcNAQELBQADggIBADaN +l8xCFWQpN5smLNb7rhVpLGsaGvdftvkHTFnq88nIua7Mui563MD1sC3AO6+fcAUR +ap8lTwEpcOPlDOHqWnzcSbvBHiqB9RZLcpHIojG5qtr8nR/zXUACE/xOHAbKsxSQ +VBcZEhrxH9cMaVr2cXj0lH2RC47skFSOvG+hTKv8dGT9cZr4QQehzZHkPJrgmzI5 +c6sq1WnIeJEmMX3ixzDx/BR4dxIOE/TdFpS/S2d7cFOFyrC78zhNLJA5wA3CXWvp +4uXViI3WLL+rG761KIcSF3Ru/H38j9CHJrAb+7lsq+KePRXBOy5nAliRn+/4Qh8s +t2j1da3Ptfb/EX3C8CSlrdP6oDyp+l3cpaDvRKS+1ujl5BOWF3sGPjLtx7dCvHaj +2GU4Kzg1USEODm8uNBNA4StnDG1KQTAYI1oyVZnJF+A83vbsea0rWBmirSwiGpWO +vpaQXUJXxPkUAzUrHC1RVwinOt4/5Mi0A3PCwSaAuwtCH60NryZy2sy+s6ODWA2C +xR9GUeOcGMyNm43sSet1UNWMKFnKdDTajAshqx7qG+XH/RU+wBeq+yNuJkbL+vmx +cmtpzyKEC2IPrNkZAJSidjzULZrtBJ4tBmIQN1IchXIbJ+XMxjHsN+xjWZsLHXbM +fjKaiJUINlK73nZfdklJrX+9ZSCyycErdhh2n1ax +-----END CERTIFICATE----- + +# Issuer: CN=Certigna Root CA O=Dhimyotis OU=0002 48146308100036 +# Subject: CN=Certigna Root CA O=Dhimyotis OU=0002 48146308100036 +# Label: "Certigna Root CA" +# Serial: 269714418870597844693661054334862075617 +# MD5 Fingerprint: 0e:5c:30:62:27:eb:5b:bc:d7:ae:62:ba:e9:d5:df:77 +# SHA1 Fingerprint: 2d:0d:52:14:ff:9e:ad:99:24:01:74:20:47:6e:6c:85:27:27:f5:43 +# SHA256 Fingerprint: d4:8d:3d:23:ee:db:50:a4:59:e5:51:97:60:1c:27:77:4b:9d:7b:18:c9:4d:5a:05:95:11:a1:02:50:b9:31:68 +-----BEGIN CERTIFICATE----- +MIIGWzCCBEOgAwIBAgIRAMrpG4nxVQMNo+ZBbcTjpuEwDQYJKoZIhvcNAQELBQAw +WjELMAkGA1UEBhMCRlIxEjAQBgNVBAoMCURoaW15b3RpczEcMBoGA1UECwwTMDAw +MiA0ODE0NjMwODEwMDAzNjEZMBcGA1UEAwwQQ2VydGlnbmEgUm9vdCBDQTAeFw0x +MzEwMDEwODMyMjdaFw0zMzEwMDEwODMyMjdaMFoxCzAJBgNVBAYTAkZSMRIwEAYD +VQQKDAlEaGlteW90aXMxHDAaBgNVBAsMEzAwMDIgNDgxNDYzMDgxMDAwMzYxGTAX +BgNVBAMMEENlcnRpZ25hIFJvb3QgQ0EwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAw +ggIKAoICAQDNGDllGlmx6mQWDoyUJJV8g9PFOSbcDO8WV43X2KyjQn+Cyu3NW9sO +ty3tRQgXstmzy9YXUnIo245Onoq2C/mehJpNdt4iKVzSs9IGPjA5qXSjklYcoW9M +CiBtnyN6tMbaLOQdLNyzKNAT8kxOAkmhVECe5uUFoC2EyP+YbNDrihqECB63aCPu +I9Vwzm1RaRDuoXrC0SIxwoKF0vJVdlB8JXrJhFwLrN1CTivngqIkicuQstDuI7pm +TLtipPlTWmR7fJj6o0ieD5Wupxj0auwuA0Wv8HT4Ks16XdG+RCYyKfHx9WzMfgIh +C59vpD++nVPiz32pLHxYGpfhPTc3GGYo0kDFUYqMwy3OU4gkWGQwFsWq4NYKpkDf +ePb1BHxpE4S80dGnBs8B92jAqFe7OmGtBIyT46388NtEbVncSVmurJqZNjBBe3Yz +IoejwpKGbvlw7q6Hh5UbxHq9MfPU0uWZ/75I7HX1eBYdpnDBfzwboZL7z8g81sWT +Co/1VTp2lc5ZmIoJlXcymoO6LAQ6l73UL77XbJuiyn1tJslV1c/DeVIICZkHJC1k +JWumIWmbat10TWuXekG9qxf5kBdIjzb5LdXF2+6qhUVB+s06RbFo5jZMm5BX7CO5 +hwjCxAnxl4YqKE3idMDaxIzb3+KhF1nOJFl0Mdp//TBt2dzhauH8XwIDAQABo4IB +GjCCARYwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYE +FBiHVuBud+4kNTxOc5of1uHieX4rMB8GA1UdIwQYMBaAFBiHVuBud+4kNTxOc5of +1uHieX4rMEQGA1UdIAQ9MDswOQYEVR0gADAxMC8GCCsGAQUFBwIBFiNodHRwczov +L3d3d3cuY2VydGlnbmEuZnIvYXV0b3JpdGVzLzBtBgNVHR8EZjBkMC+gLaArhilo +dHRwOi8vY3JsLmNlcnRpZ25hLmZyL2NlcnRpZ25hcm9vdGNhLmNybDAxoC+gLYYr +aHR0cDovL2NybC5kaGlteW90aXMuY29tL2NlcnRpZ25hcm9vdGNhLmNybDANBgkq +hkiG9w0BAQsFAAOCAgEAlLieT/DjlQgi581oQfccVdV8AOItOoldaDgvUSILSo3L +6btdPrtcPbEo/uRTVRPPoZAbAh1fZkYJMyjhDSSXcNMQH+pkV5a7XdrnxIxPTGRG +HVyH41neQtGbqH6mid2PHMkwgu07nM3A6RngatgCdTer9zQoKJHyBApPNeNgJgH6 +0BGM+RFq7q89w1DTj18zeTyGqHNFkIwgtnJzFyO+B2XleJINugHA64wcZr+shncB +lA2c5uk5jR+mUYyZDDl34bSb+hxnV29qao6pK0xXeXpXIs/NX2NGjVxZOob4Mkdi +o2cNGJHc+6Zr9UhhcyNZjgKnvETq9Emd8VRY+WCv2hikLyhF3HqgiIZd8zvn/yk1 +gPxkQ5Tm4xxvvq0OKmOZK8l+hfZx6AYDlf7ej0gcWtSS6Cvu5zHbugRqh5jnxV/v +faci9wHYTfmJ0A6aBVmknpjZbyvKcL5kwlWj9Omvw5Ip3IgWJJk8jSaYtlu3zM63 +Nwf9JtmYhST/WSMDmu2dnajkXjjO11INb9I/bbEFa0nOipFGc/T2L/Coc3cOZayh +jWZSaX5LaAzHHjcng6WMxwLkFM1JAbBzs/3GkDpv0mztO+7skb6iQ12LAEpmJURw +3kAP+HwV96LOPNdeE4yBFxgX0b3xdxA61GU5wSesVywlVP+i2k+KYTlerj1KjL0= +-----END CERTIFICATE----- + +# Issuer: CN=emSign Root CA - G1 O=eMudhra Technologies Limited OU=emSign PKI +# Subject: CN=emSign Root CA - G1 O=eMudhra Technologies Limited OU=emSign PKI +# Label: "emSign Root CA - G1" +# Serial: 235931866688319308814040 +# MD5 Fingerprint: 9c:42:84:57:dd:cb:0b:a7:2e:95:ad:b6:f3:da:bc:ac +# SHA1 Fingerprint: 8a:c7:ad:8f:73:ac:4e:c1:b5:75:4d:a5:40:f4:fc:cf:7c:b5:8e:8c +# SHA256 Fingerprint: 40:f6:af:03:46:a9:9a:a1:cd:1d:55:5a:4e:9c:ce:62:c7:f9:63:46:03:ee:40:66:15:83:3d:c8:c8:d0:03:67 +-----BEGIN CERTIFICATE----- +MIIDlDCCAnygAwIBAgIKMfXkYgxsWO3W2DANBgkqhkiG9w0BAQsFADBnMQswCQYD +VQQGEwJJTjETMBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhyYSBU +ZWNobm9sb2dpZXMgTGltaXRlZDEcMBoGA1UEAxMTZW1TaWduIFJvb3QgQ0EgLSBH +MTAeFw0xODAyMTgxODMwMDBaFw00MzAyMTgxODMwMDBaMGcxCzAJBgNVBAYTAklO +MRMwEQYDVQQLEwplbVNpZ24gUEtJMSUwIwYDVQQKExxlTXVkaHJhIFRlY2hub2xv +Z2llcyBMaW1pdGVkMRwwGgYDVQQDExNlbVNpZ24gUm9vdCBDQSAtIEcxMIIBIjAN +BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAk0u76WaK7p1b1TST0Bsew+eeuGQz +f2N4aLTNLnF115sgxk0pvLZoYIr3IZpWNVrzdr3YzZr/k1ZLpVkGoZM0Kd0WNHVO +8oG0x5ZOrRkVUkr+PHB1cM2vK6sVmjM8qrOLqs1D/fXqcP/tzxE7lM5OMhbTI0Aq +d7OvPAEsbO2ZLIvZTmmYsvePQbAyeGHWDV/D+qJAkh1cF+ZwPjXnorfCYuKrpDhM +tTk1b+oDafo6VGiFbdbyL0NVHpENDtjVaqSW0RM8LHhQ6DqS0hdW5TUaQBw+jSzt +Od9C4INBdN+jzcKGYEho42kLVACL5HZpIQ15TjQIXhTCzLG3rdd8cIrHhQIDAQAB +o0IwQDAdBgNVHQ4EFgQU++8Nhp6w492pufEhF38+/PB3KxowDgYDVR0PAQH/BAQD +AgEGMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAFn/8oz1h31x +PaOfG1vR2vjTnGs2vZupYeveFix0PZ7mddrXuqe8QhfnPZHr5X3dPpzxz5KsbEjM +wiI/aTvFthUvozXGaCocV685743QNcMYDHsAVhzNixl03r4PEuDQqqE/AjSxcM6d +GNYIAwlG7mDgfrbESQRRfXBgvKqy/3lyeqYdPV8q+Mri/Tm3R7nrft8EI6/6nAYH +6ftjk4BAtcZsCjEozgyfz7MjNYBBjWzEN3uBL4ChQEKF6dk4jeihU80Bv2noWgby +RQuQ+q7hv53yrlc8pa6yVvSLZUDp/TGBLPQ5Cdjua6e0ph0VpZj3AYHYhX3zUVxx +iN66zB+Afko= +-----END CERTIFICATE----- + +# Issuer: CN=emSign ECC Root CA - G3 O=eMudhra Technologies Limited OU=emSign PKI +# Subject: CN=emSign ECC Root CA - G3 O=eMudhra Technologies Limited OU=emSign PKI +# Label: "emSign ECC Root CA - G3" +# Serial: 287880440101571086945156 +# MD5 Fingerprint: ce:0b:72:d1:9f:88:8e:d0:50:03:e8:e3:b8:8b:67:40 +# SHA1 Fingerprint: 30:43:fa:4f:f2:57:dc:a0:c3:80:ee:2e:58:ea:78:b2:3f:e6:bb:c1 +# SHA256 Fingerprint: 86:a1:ec:ba:08:9c:4a:8d:3b:be:27:34:c6:12:ba:34:1d:81:3e:04:3c:f9:e8:a8:62:cd:5c:57:a3:6b:be:6b +-----BEGIN CERTIFICATE----- +MIICTjCCAdOgAwIBAgIKPPYHqWhwDtqLhDAKBggqhkjOPQQDAzBrMQswCQYDVQQG +EwJJTjETMBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhyYSBUZWNo +bm9sb2dpZXMgTGltaXRlZDEgMB4GA1UEAxMXZW1TaWduIEVDQyBSb290IENBIC0g +RzMwHhcNMTgwMjE4MTgzMDAwWhcNNDMwMjE4MTgzMDAwWjBrMQswCQYDVQQGEwJJ +TjETMBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhyYSBUZWNobm9s +b2dpZXMgTGltaXRlZDEgMB4GA1UEAxMXZW1TaWduIEVDQyBSb290IENBIC0gRzMw +djAQBgcqhkjOPQIBBgUrgQQAIgNiAAQjpQy4LRL1KPOxst3iAhKAnjlfSU2fySU0 +WXTsuwYc58Byr+iuL+FBVIcUqEqy6HyC5ltqtdyzdc6LBtCGI79G1Y4PPwT01xyS +fvalY8L1X44uT6EYGQIrMgqCZH0Wk9GjQjBAMB0GA1UdDgQWBBR8XQKEE9TMipuB +zhccLikenEhjQjAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAKBggq +hkjOPQQDAwNpADBmAjEAvvNhzwIQHWSVB7gYboiFBS+DCBeQyh+KTOgNG3qxrdWB +CUfvO6wIBHxcmbHtRwfSAjEAnbpV/KlK6O3t5nYBQnvI+GDZjVGLVTv7jHvrZQnD ++JbNR6iC8hZVdyR+EhCVBCyj +-----END CERTIFICATE----- + +# Issuer: CN=emSign Root CA - C1 O=eMudhra Inc OU=emSign PKI +# Subject: CN=emSign Root CA - C1 O=eMudhra Inc OU=emSign PKI +# Label: "emSign Root CA - C1" +# Serial: 825510296613316004955058 +# MD5 Fingerprint: d8:e3:5d:01:21:fa:78:5a:b0:df:ba:d2:ee:2a:5f:68 +# SHA1 Fingerprint: e7:2e:f1:df:fc:b2:09:28:cf:5d:d4:d5:67:37:b1:51:cb:86:4f:01 +# SHA256 Fingerprint: 12:56:09:aa:30:1d:a0:a2:49:b9:7a:82:39:cb:6a:34:21:6f:44:dc:ac:9f:39:54:b1:42:92:f2:e8:c8:60:8f +-----BEGIN CERTIFICATE----- +MIIDczCCAlugAwIBAgILAK7PALrEzzL4Q7IwDQYJKoZIhvcNAQELBQAwVjELMAkG +A1UEBhMCVVMxEzARBgNVBAsTCmVtU2lnbiBQS0kxFDASBgNVBAoTC2VNdWRocmEg +SW5jMRwwGgYDVQQDExNlbVNpZ24gUm9vdCBDQSAtIEMxMB4XDTE4MDIxODE4MzAw +MFoXDTQzMDIxODE4MzAwMFowVjELMAkGA1UEBhMCVVMxEzARBgNVBAsTCmVtU2ln +biBQS0kxFDASBgNVBAoTC2VNdWRocmEgSW5jMRwwGgYDVQQDExNlbVNpZ24gUm9v +dCBDQSAtIEMxMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAz+upufGZ +BczYKCFK83M0UYRWEPWgTywS4/oTmifQz/l5GnRfHXk5/Fv4cI7gklL35CX5VIPZ +HdPIWoU/Xse2B+4+wM6ar6xWQio5JXDWv7V7Nq2s9nPczdcdioOl+yuQFTdrHCZH +3DspVpNqs8FqOp099cGXOFgFixwR4+S0uF2FHYP+eF8LRWgYSKVGczQ7/g/IdrvH +GPMF0Ybzhe3nudkyrVWIzqa2kbBPrH4VI5b2P/AgNBbeCsbEBEV5f6f9vtKppa+c +xSMq9zwhbL2vj07FOrLzNBL834AaSaTUqZX3noleoomslMuoaJuvimUnzYnu3Yy1 +aylwQ6BpC+S5DwIDAQABo0IwQDAdBgNVHQ4EFgQU/qHgcB4qAzlSWkK+XJGFehiq +TbUwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQEL +BQADggEBAMJKVvoVIXsoounlHfv4LcQ5lkFMOycsxGwYFYDGrK9HWS8mC+M2sO87 +/kOXSTKZEhVb3xEp/6tT+LvBeA+snFOvV71ojD1pM/CjoCNjO2RnIkSt1XHLVip4 +kqNPEjE2NuLe/gDEo2APJ62gsIq1NnpSob0n9CAnYuhNlCQT5AoE6TyrLshDCUrG +YQTlSTR+08TI9Q/Aqum6VF7zYytPT1DU/rl7mYw9wC68AivTxEDkigcxHpvOJpkT ++xHqmiIMERnHXhuBUDDIlhJu58tBf5E7oke3VIAb3ADMmpDqw8NQBmIMMMAVSKeo +WXzhriKi4gp6D/piq1JM4fHfyr6DDUI= +-----END CERTIFICATE----- + +# Issuer: CN=emSign ECC Root CA - C3 O=eMudhra Inc OU=emSign PKI +# Subject: CN=emSign ECC Root CA - C3 O=eMudhra Inc OU=emSign PKI +# Label: "emSign ECC Root CA - C3" +# Serial: 582948710642506000014504 +# MD5 Fingerprint: 3e:53:b3:a3:81:ee:d7:10:f8:d3:b0:1d:17:92:f5:d5 +# SHA1 Fingerprint: b6:af:43:c2:9b:81:53:7d:f6:ef:6b:c3:1f:1f:60:15:0c:ee:48:66 +# SHA256 Fingerprint: bc:4d:80:9b:15:18:9d:78:db:3e:1d:8c:f4:f9:72:6a:79:5d:a1:64:3c:a5:f1:35:8e:1d:db:0e:dc:0d:7e:b3 +-----BEGIN CERTIFICATE----- +MIICKzCCAbGgAwIBAgIKe3G2gla4EnycqDAKBggqhkjOPQQDAzBaMQswCQYDVQQG +EwJVUzETMBEGA1UECxMKZW1TaWduIFBLSTEUMBIGA1UEChMLZU11ZGhyYSBJbmMx +IDAeBgNVBAMTF2VtU2lnbiBFQ0MgUm9vdCBDQSAtIEMzMB4XDTE4MDIxODE4MzAw +MFoXDTQzMDIxODE4MzAwMFowWjELMAkGA1UEBhMCVVMxEzARBgNVBAsTCmVtU2ln +biBQS0kxFDASBgNVBAoTC2VNdWRocmEgSW5jMSAwHgYDVQQDExdlbVNpZ24gRUND +IFJvb3QgQ0EgLSBDMzB2MBAGByqGSM49AgEGBSuBBAAiA2IABP2lYa57JhAd6bci +MK4G9IGzsUJxlTm801Ljr6/58pc1kjZGDoeVjbk5Wum739D+yAdBPLtVb4Ojavti +sIGJAnB9SMVK4+kiVCJNk7tCDK93nCOmfddhEc5lx/h//vXyqaNCMEAwHQYDVR0O +BBYEFPtaSNCAIEDyqOkAB2kZd6fmw/TPMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMB +Af8EBTADAQH/MAoGCCqGSM49BAMDA2gAMGUCMQC02C8Cif22TGK6Q04ThHK1rt0c +3ta13FaPWEBaLd4gTCKDypOofu4SQMfWh0/434UCMBwUZOR8loMRnLDRWmFLpg9J +0wD8ofzkpf9/rdcw0Md3f76BB1UwUCAU9Vc4CqgxUQ== +-----END CERTIFICATE----- + +# Issuer: CN=Hongkong Post Root CA 3 O=Hongkong Post +# Subject: CN=Hongkong Post Root CA 3 O=Hongkong Post +# Label: "Hongkong Post Root CA 3" +# Serial: 46170865288971385588281144162979347873371282084 +# MD5 Fingerprint: 11:fc:9f:bd:73:30:02:8a:fd:3f:f3:58:b9:cb:20:f0 +# SHA1 Fingerprint: 58:a2:d0:ec:20:52:81:5b:c1:f3:f8:64:02:24:4e:c2:8e:02:4b:02 +# SHA256 Fingerprint: 5a:2f:c0:3f:0c:83:b0:90:bb:fa:40:60:4b:09:88:44:6c:76:36:18:3d:f9:84:6e:17:10:1a:44:7f:b8:ef:d6 +-----BEGIN CERTIFICATE----- +MIIFzzCCA7egAwIBAgIUCBZfikyl7ADJk0DfxMauI7gcWqQwDQYJKoZIhvcNAQEL +BQAwbzELMAkGA1UEBhMCSEsxEjAQBgNVBAgTCUhvbmcgS29uZzESMBAGA1UEBxMJ +SG9uZyBLb25nMRYwFAYDVQQKEw1Ib25na29uZyBQb3N0MSAwHgYDVQQDExdIb25n +a29uZyBQb3N0IFJvb3QgQ0EgMzAeFw0xNzA2MDMwMjI5NDZaFw00MjA2MDMwMjI5 +NDZaMG8xCzAJBgNVBAYTAkhLMRIwEAYDVQQIEwlIb25nIEtvbmcxEjAQBgNVBAcT +CUhvbmcgS29uZzEWMBQGA1UEChMNSG9uZ2tvbmcgUG9zdDEgMB4GA1UEAxMXSG9u +Z2tvbmcgUG9zdCBSb290IENBIDMwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIK +AoICAQCziNfqzg8gTr7m1gNt7ln8wlffKWihgw4+aMdoWJwcYEuJQwy51BWy7sFO +dem1p+/l6TWZ5Mwc50tfjTMwIDNT2aa71T4Tjukfh0mtUC1Qyhi+AViiE3CWu4mI +VoBc+L0sPOFMV4i707mV78vH9toxdCim5lSJ9UExyuUmGs2C4HDaOym71QP1mbpV +9WTRYA6ziUm4ii8F0oRFKHyPaFASePwLtVPLwpgchKOesL4jpNrcyCse2m5FHomY +2vkALgbpDDtw1VAliJnLzXNg99X/NWfFobxeq81KuEXryGgeDQ0URhLj0mRiikKY +vLTGCAj4/ahMZJx2Ab0vqWwzD9g/KLg8aQFChn5pwckGyuV6RmXpwtZQQS4/t+Tt +bNe/JgERohYpSms0BpDsE9K2+2p20jzt8NYt3eEV7KObLyzJPivkaTv/ciWxNoZb +x39ri1UbSsUgYT2uy1DhCDq+sI9jQVMwCFk8mB13umOResoQUGC/8Ne8lYePl8X+ +l2oBlKN8W4UdKjk60FSh0Tlxnf0h+bV78OLgAo9uliQlLKAeLKjEiafv7ZkGL7YK +TE/bosw3Gq9HhS2KX8Q0NEwA/RiTZxPRN+ZItIsGxVd7GYYKecsAyVKvQv83j+Gj +Hno9UKtjBucVtT+2RTeUN7F+8kjDf8V1/peNRY8apxpyKBpADwIDAQABo2MwYTAP +BgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBBjAfBgNVHSMEGDAWgBQXnc0e +i9Y5K3DTXNSguB+wAPzFYTAdBgNVHQ4EFgQUF53NHovWOStw01zUoLgfsAD8xWEw +DQYJKoZIhvcNAQELBQADggIBAFbVe27mIgHSQpsY1Q7XZiNc4/6gx5LS6ZStS6LG +7BJ8dNVI0lkUmcDrudHr9EgwW62nV3OZqdPlt9EuWSRY3GguLmLYauRwCy0gUCCk +MpXRAJi70/33MvJJrsZ64Ee+bs7Lo3I6LWldy8joRTnU+kLBEUx3XZL7av9YROXr +gZ6voJmtvqkBZss4HTzfQx/0TW60uhdG/H39h4F5ag0zD/ov+BS5gLNdTaqX4fnk +GMX41TiMJjz98iji7lpJiCzfeT2OnpA8vUFKOt1b9pq0zj8lMH8yfaIDlNDceqFS +3m6TjRgm/VWsvY+b0s+v54Ysyx8Jb6NvqYTUc79NoXQbTiNg8swOqn+knEwlqLJm +Ozj/2ZQw9nKEvmhVEA/GcywWaZMH/rFF7buiVWqw2rVKAiUnhde3t4ZEFolsgCs+ +l6mc1X5VTMbeRRAc6uk7nwNT7u56AQIWeNTowr5GdogTPyK7SBIdUgC0An4hGh6c +JfTzPV4e0hz5sy229zdcxsshTrD3mUcYhcErulWuBurQB7Lcq9CClnXO0lD+mefP +L5/ndtFhKvshuzHQqp9HpLIiyhY6UFfEW0NnxWViA0kB60PZ2Pierc+xYw5F9KBa +LJstxabArahH9CdMOA0uG0k7UvToiIMrVCjU8jVStDKDYmlkDJGcn5fqdBb9HxEG +mpv0 +-----END CERTIFICATE----- diff --git a/libs/certifi/core.py b/libs/certifi/core.py index eab9d1d17..7271acf40 100644 --- a/libs/certifi/core.py +++ b/libs/certifi/core.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- """ @@ -8,30 +7,9 @@ certifi.py This module returns the installation location of cacert.pem. """ import os -import warnings - - -class DeprecatedBundleWarning(DeprecationWarning): - """ - The weak security bundle is being deprecated. Please bother your service - provider to get them to stop using cross-signed roots. - """ def where(): f = os.path.dirname(__file__) return os.path.join(f, 'cacert.pem') - - -def old_where(): - warnings.warn( - "The weak security bundle has been removed. certifi.old_where() is now an alias " - "of certifi.where(). Please update your code to use certifi.where() instead. " - "certifi.old_where() will be removed in 2018.", - DeprecatedBundleWarning - ) - return where() - -if __name__ == '__main__': - print(where()) diff --git a/libs/cfscrape/__init__.py b/libs/cfscrape/__init__.py index e1d0c8fc8..c31bb377b 100644 --- a/libs/cfscrape/__init__.py +++ b/libs/cfscrape/__init__.py @@ -1,7 +1,10 @@ +# coding=utf-8 + import logging import random import re - +import os +import json import base64 from copy import deepcopy @@ -11,6 +14,7 @@ from .jsfuck import jsunfuck import js2py from requests.sessions import Session +from subliminal_patch.pitcher import pitchers try: from requests_toolbelt.utils import dump @@ -24,6 +28,15 @@ except ImportError: from urllib.parse import urlparse from urllib.parse import urlunparse +brotli_available = True + +try: + from brotli import decompress as brdec +except: + brotli_available = False + +logger = logging.getLogger(__name__) + __version__ = "2.0.3" # Orignally written by https://github.com/Anorov/cloudflare-scrape @@ -43,16 +56,47 @@ BUG_REPORT = """\ Cloudflare may have changed their technique, or there may be a bug in the script. """ + +cur_path = os.path.abspath(os.path.dirname(__file__)) + +if brotli_available: + brwsrs = os.path.join(cur_path, "browsers_br.json") + with open(brwsrs, "r") as f: + UA_COMBO = json.load(f, object_pairs_hook=OrderedDict)["chrome"] + +else: + brwsrs = os.path.join(cur_path, "browsers.json") + UA_COMBO = [] + with open(brwsrs, "r") as f: + _brwsrs = json.load(f, object_pairs_hook=OrderedDict) + for entry in _brwsrs: + _entry = OrderedDict(("-".join(a.capitalize() for a in key.split("-")), value) + for key, value in entry.iteritems()) + _entry["User-Agent"] = None + UA_COMBO.append({"User-Agent": [entry["user-agent"]], "headers": _entry}) + + +class NeedsCaptchaException(Exception): + pass + + class CloudflareScraper(Session): def __init__(self, *args, **kwargs): self.delay = kwargs.pop('delay', 8) self.debug = False + self._was_cf = False + self._ua = None + self._hdrs = None super(CloudflareScraper, self).__init__(*args, **kwargs) - if 'requests' in self.headers['User-Agent']: + if not self._ua: # Set a random User-Agent if no custom User-Agent has been set - self.headers['User-Agent'] = random.choice(DEFAULT_USER_AGENTS) + ua_combo = random.choice(UA_COMBO) + self._ua = random.choice(ua_combo["User-Agent"]) + self._hdrs = ua_combo["headers"].copy() + self._hdrs["User-Agent"] = self._ua + self.headers['User-Agent'] = self._ua def set_cloudflare_challenge_delay(self, delay): if isinstance(delay, (int, float)) and delay > 0: @@ -61,7 +105,7 @@ class CloudflareScraper(Session): def is_cloudflare_challenge(self, resp): if resp.headers.get('Server', '').startswith('cloudflare'): if b'why_captcha' in resp.content or b'/cdn-cgi/l/chk_captcha' in resp.content: - raise ValueError('Captcha') + raise NeedsCaptchaException return ( resp.status_code in [429, 503] @@ -77,34 +121,75 @@ class CloudflareScraper(Session): pass def request(self, method, url, *args, **kwargs): - if not isinstance(self.headers, OrderedDict): - self.headers = \ - OrderedDict( - [ - ('User-Agent', self.headers['User-Agent']), - ('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'), - ('Accept-Language', 'en-US,en;q=0.5'), - ('Accept-Encoding', 'gzip, deflate'), - ('Connection', 'close'), - ('Upgrade-Insecure-Requests', '1') - ] - ) + # self.headers = ( + # OrderedDict( + # [ + # ('User-Agent', self.headers['User-Agent']), + # ('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'), + # ('Accept-Language', 'en-US,en;q=0.5'), + # ('Accept-Encoding', 'gzip, deflate'), + # ('Connection', 'close'), + # ('Upgrade-Insecure-Requests', '1') + # ] + # ) + # ) + self.headers = self._hdrs.copy() resp = super(CloudflareScraper, self).request(method, url, *args, **kwargs) + if resp.headers.get('content-encoding') == 'br' and brotli_available: + resp._content = brdec(resp._content) # Debug request if self.debug: self.debugRequest(resp) # Check if Cloudflare anti-bot is on - if self.is_cloudflare_challenge(resp): - # Work around if the initial request is not a GET, - # Superseed with a GET then re-request the orignal METHOD. - if resp.request.method != 'GET': - self.request('GET', resp.url) - resp = self.request(method, url, *args, **kwargs) - else: - resp = self.solve_cf_challenge(resp, **kwargs) + try: + if self.is_cloudflare_challenge(resp): + self._was_cf = True + # Work around if the initial request is not a GET, + # Superseed with a GET then re-request the orignal METHOD. + if resp.request.method != 'GET': + self.request('GET', resp.url) + resp = self.request(method, url, *args, **kwargs) + else: + resp = self.solve_cf_challenge(resp, **kwargs) + except NeedsCaptchaException: + # solve the captcha + self._was_cf = True + site_key = re.search(r'data-sitekey="(.+?)"', resp.content).group(1) + challenge_s = re.search(r'type="hidden" name="s" value="(.+?)"', resp.content).group(1) + challenge_ray = re.search(r'data-ray="(.+?)"', resp.content).group(1) + if not all([site_key, challenge_s, challenge_ray]): + raise Exception("cf: Captcha site-key not found!") + + pitcher = pitchers.get_pitcher()("cf", resp.request.url, site_key, + user_agent=self.headers["User-Agent"], + cookies=self.cookies.get_dict(), + is_invisible=True) + + parsed_url = urlparse(resp.url) + domain = parsed_url.netloc + logger.info("cf: %s: Solving captcha", domain) + result = pitcher.throw() + if not result: + raise Exception("cf: Couldn't solve captcha!") + + submit_url = '{}://{}/cdn-cgi/l/chk_captcha'.format(parsed_url.scheme, domain) + method = resp.request.method + + cloudflare_kwargs = { + 'allow_redirects': False, + 'headers': {'Referer': resp.url}, + 'params': OrderedDict( + [ + ('s', challenge_s), + ('g-recaptcha-response', result) + ] + ) + } + + return self.request(method, submit_url, **cloudflare_kwargs) return resp @@ -127,7 +212,7 @@ class CloudflareScraper(Session): submit_url = '{}://{}/cdn-cgi/l/chk_jschl'.format(parsed_url.scheme, domain) cloudflare_kwargs = deepcopy(original_kwargs) - headers = cloudflare_kwargs.setdefault('headers', OrderedDict({'Referer': resp.url})) + headers = cloudflare_kwargs.setdefault('headers', {'Referer': resp.url}) try: params = cloudflare_kwargs.setdefault( @@ -297,22 +382,6 @@ class CloudflareScraper(Session): scraper.headers['User-Agent'] ) - def get_live_tokens(self, domain): - for d in self.cookies.list_domains(): - if d.startswith(".") and d in ("." + domain): - cookie_domain = d - break - else: - raise ValueError( - "Unable to find Cloudflare cookies. Does the site actually have Cloudflare IUAM (\"I'm Under Attack Mode\") enabled?") - - return ({ - "__cfduid": self.cookies.get("__cfduid", "", domain=cookie_domain), - "cf_clearance": self.cookies.get("cf_clearance", "", domain=cookie_domain) - }, - self.headers["User-Agent"] - ) - @classmethod def get_cookie_string(cls, url, user_agent=None, debug=False, **kwargs): """ diff --git a/libs/cfscrape/browsers.json b/libs/cfscrape/browsers.json new file mode 100644 index 000000000..f1b7ad3c9 --- /dev/null +++ b/libs/cfscrape/browsers.json @@ -0,0 +1,80 @@ +[ + { + "connection": "close", + "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", + "user-agent": "Mozilla/5.0 (Windows NT 5.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.102 Safari/537.36", + "accept-encoding": "gzip,deflate", + "accept-language": "en-US,en;q=0.8" + }, + { + "connection": "close", + "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", + "user-agent": "Mozilla/5.0 (Windows NT 5.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.101 Safari/537.36", + "accept-encoding": "gzip,deflate", + "accept-language": "en-US,en;q=0.8" + }, + { + "connection": "close", + "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", + "upgrade-insecure-requests": "1", + "user-agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.97 Safari/537.36", + "accept-language": "en-US,en;q=0.8", + "accept-encoding": "gzip, deflate, " + }, + { + "connection": "close", + "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", + "upgrade-insecure-requests": "1", + "user-agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.75 Safari/537.36", + "accept-language": "en-US,en;q=0.8", + "accept-encoding": "gzip, deflate, " + }, + { + "connection": "close", + "accept": "*/*", + "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:30.0) Gecko/20100101 Firefox/30.0" + }, + { + "connection": "close", + "accept": "image/jpeg, image/gif, image/pjpeg, application/x-ms-application, application/xaml+xml, application/x-ms-xbap, */*", + "accept-language": "en-US", + "user-agent": "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET4.0C; .NET4.0E)", + "accept-encoding": "gzip, deflate" + }, + { + "connection": "close", + "accept": "text/html, application/xhtml+xml, */*", + "accept-language": "en-US", + "user-agent": "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)", + "accept-encoding": "gzip, deflate" + }, + { + "connection": "close", + "accept": "text/html, application/xhtml+xml, */*", + "accept-language": "en-US", + "user-agent": "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)", + "accept-encoding": "gzip, deflate", + "dnt": "1" + }, + { + "connection": "close", + "user-agent": "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0", + "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "accept-language": "en-US,en;q=0.5", + "accept-encoding": "gzip, deflate" + }, + { + "connection": "close", + "user-agent": "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:42.0) Gecko/20100101 Firefox/42.0", + "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "accept-language": "en-US,en;q=0.5", + "accept-encoding": "gzip, deflate" + }, + { + "connection": "close", + "user-agent": "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0", + "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "accept-language": "en-US,en;q=0.5", + "accept-encoding": "gzip, deflate" + } +] \ No newline at end of file diff --git a/libs/cfscrape/browsers_br.json b/libs/cfscrape/browsers_br.json new file mode 100644 index 000000000..c3eab3155 --- /dev/null +++ b/libs/cfscrape/browsers_br.json @@ -0,0 +1,336 @@ +{ + "chrome": [ + { + "User-Agent": [ + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.82 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.82 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.82 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.82 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.82 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.82 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.82 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.82 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.82 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.82 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.101 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.101 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.101 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.75 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.75 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.75 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.75 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.75 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.75 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.75 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.75 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.75 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.75 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.110 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.110 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.110 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.110 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.81 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.81 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.81 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.81 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.81 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.81 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.81 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.81 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.81 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.81 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.113 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.89 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.89 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.89 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.89 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.89 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.89 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.76 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.76 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.76 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.76 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.76 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.76 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36" + ], + "headers": { + "Connection": "keep-alive", + "Upgrade-Insecure-Requests": "1", + "User-Agent": null, + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.8", + "Accept-Encoding": "gzip, deflate, , br" + } + }, + { + "User-Agent": [ + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.86 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.86 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.86 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.86 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.86 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.86 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.86 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.86 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.86 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.86 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.78 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.78 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.78 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.78 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.78 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.78 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.78 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.78 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.78 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.78 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36" + ], + "headers": { + "Connection": "keep-alive", + "Upgrade-Insecure-Requests": "1", + "User-Agent": null, + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.8", + "Accept-Encoding": "gzip, deflate, br" + } + }, + { + "User-Agent": [ + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.62 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.62 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.62 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.62 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.62 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.62 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.62 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.62 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.62 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.62 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.119 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.119 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.119 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.119 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.119 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.119 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.119 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.119 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.119 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.119 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.146 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.146 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.146 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.146 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.146 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.146 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.146 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.146 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.146 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.146 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.117 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.117 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.117 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.117 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.117 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.117 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.117 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.117 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.117 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.67 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.67 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.67 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.67 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.67 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.67 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.67 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.67 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.67 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.62 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.62 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.62 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.62 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.62 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.62 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.62 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.62 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.62 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.92 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.92 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.109 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.170 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.75 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.75 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.75 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.75 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.75 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.81 Safari/537.36" + ], + "headers": { + "Connection": "keep-alive", + "Upgrade-Insecure-Requests": "1", + "User-Agent": null, + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.9", + "Accept-Encoding": "gzip, deflate, br" + } + }, + { + "User-Agent": [ + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36" + ], + "headers": { + "Connection": "keep-alive", + "User-Agent": null, + "Upgrade-Insecure-Requests": "1", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.9", + "Accept-Encoding": "gzip, deflate, br" + } + }, + { + "User-Agent": [ + "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.40 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.40 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.28 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.28 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.28 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.28 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.28 Safari/537.36" + ], + "headers": { + "Connection": "keep-alive", + "Upgrade-Insecure-Requests": "1", + "User-Agent": null, + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3", + "Accept-Language": "en-US,en;q=0.9", + "Accept-Encoding": "gzip, deflate, br" + } + }, + { + "User-Agent": [ + "Mozilla/5.0 (Linux; Android 8.1.0; SM-N960F Build/M1AJQ) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.85 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 8.0.0; SM-G965F Build/R16NW) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.85 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 8.0.0; Pixel 2 Build/OPD1.170816.010) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.85 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 8.0.0; Pixel Build/OPR6.170623.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.85 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 7.1.1; SM-A530F Build/NMF26X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.85 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 7.1; Pixel Build/NDE63H) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.85 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 7.0; SM-G955F Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.85 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 7.0; SM-G950F Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.85 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 7.0; SM-T825 Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.85 Safari/537.36", + "Mozilla/5.0 (Linux; Android 6.0.1; SM-G930F Build/MMB29K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.85 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 6.0; Nexus 6 Build/MRA58K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.85 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 6.0; XT1092 Build/MPE24.49-18) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.85 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 6.0.1; SM-N910C Build/MMB29K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.85 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 5.0.2; SM-G920F Build/LRX22G) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.85 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 5.0; Nexus 6 Build/LRX21O) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.85 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 9; Pixel 3 XL Build/PD1A.180720.030) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.85 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 9; Pixel 3 Build/PD1A.180720.030) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.85 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 9; Pixel 2 Build/PPR1.180610.009) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.85 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 4.4; Nexus 5 Build/KRT16M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.85 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 4.4.2; SM-T530 Build/KOT49H) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.85 Safari/537.36", + "Mozilla/5.0 (Linux; Android 4.4.4; SM-N910C Build/KTU84P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.85 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 5.1.1; Nexus 9 Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.85 Safari/537.36", + "Mozilla/5.0 (Linux; Android 7.1.1; SM-N950F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.90 Mobile Safari/537.36" + ], + "headers": { + "Connection": "keep-alive", + "Upgrade-Insecure-Requests": "1", + "User-Agent": null, + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8", + "Accept-Encoding": "gzip, deflate, br", + "Accept-Language": "en-US,en;q=0.9" + } + }, + { + "User-Agent": [ + "Mozilla/5.0 (Linux; Android 8.1.0; SM-T835 Build/M1AJQ) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.85 Safari/537.36", + "Mozilla/5.0 (Linux; Android 8.0.0; SM-G960F Build/R16NW) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.85 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 5.0; XT1092 Build/LXE22.46-19) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.85 Mobile Safari/537.36" + ], + "headers": { + "Connection": "keep-alive", + "Upgrade-Insecure-Requests": "1", + "User-Agent": null, + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8", + "Accept-Encoding": "gzip, deflate, br", + "Accept-Language": "en-GB,en-US;q=0.9,en;q=0.8" + } + } + ] +} diff --git a/libs/requests/__version__.py b/libs/requests/__version__.py index be8a45fe0..f5b5d0367 100644 --- a/libs/requests/__version__.py +++ b/libs/requests/__version__.py @@ -5,8 +5,8 @@ __title__ = 'requests' __description__ = 'Python HTTP for Humans.' __url__ = 'http://python-requests.org' -__version__ = '2.20.0' -__build__ = 0x022000 +__version__ = '2.21.0' +__build__ = 0x022100 __author__ = 'Kenneth Reitz' __author_email__ = 'me@kennethreitz.org' __license__ = 'Apache 2.0' diff --git a/libs/requests/models.py b/libs/requests/models.py index 3dded57ef..62dcd0b7c 100644 --- a/libs/requests/models.py +++ b/libs/requests/models.py @@ -781,7 +781,7 @@ class Response(object): return chunks - def iter_lines(self, chunk_size=ITER_CHUNK_SIZE, decode_unicode=None, delimiter=None): + def iter_lines(self, chunk_size=ITER_CHUNK_SIZE, decode_unicode=False, delimiter=None): """Iterates over the response data, one line at a time. When stream=True is set on the request, this avoids reading the content at once into memory for large responses. diff --git a/libs/requests/sessions.py b/libs/requests/sessions.py index a448bd83f..d73d700fa 100644 --- a/libs/requests/sessions.py +++ b/libs/requests/sessions.py @@ -19,7 +19,7 @@ from .cookies import ( from .models import Request, PreparedRequest, DEFAULT_REDIRECT_LIMIT from .hooks import default_hooks, dispatch_hook from ._internal_utils import to_native_string -from .utils import to_key_val_list, default_headers +from .utils import to_key_val_list, default_headers, DEFAULT_PORTS from .exceptions import ( TooManyRedirects, InvalidSchema, ChunkedEncodingError, ContentDecodingError) @@ -128,8 +128,17 @@ class SessionRedirectMixin(object): if (old_parsed.scheme == 'http' and old_parsed.port in (80, None) and new_parsed.scheme == 'https' and new_parsed.port in (443, None)): return False + + # Handle default port usage corresponding to scheme. + changed_port = old_parsed.port != new_parsed.port + changed_scheme = old_parsed.scheme != new_parsed.scheme + default_port = (DEFAULT_PORTS.get(old_parsed.scheme, None), None) + if (not changed_scheme and old_parsed.port in default_port + and new_parsed.port in default_port): + return False + # Standard case: root URI must match - return old_parsed.port != new_parsed.port or old_parsed.scheme != new_parsed.scheme + return changed_port or changed_scheme def resolve_redirects(self, resp, req, stream=False, timeout=None, verify=True, cert=None, proxies=None, yield_requests=False, **adapter_kwargs): diff --git a/libs/requests/utils.py b/libs/requests/utils.py index 0ce7fe115..8170a8d2c 100644 --- a/libs/requests/utils.py +++ b/libs/requests/utils.py @@ -38,6 +38,8 @@ NETRC_FILES = ('.netrc', '_netrc') DEFAULT_CA_BUNDLE_PATH = certs.where() +DEFAULT_PORTS = {'http': 80, 'https': 443} + if sys.platform == 'win32': # provide a proxy_bypass version on Windows without DNS lookups @@ -264,7 +266,7 @@ def from_key_val_list(value): >>> from_key_val_list([('key', 'val')]) OrderedDict([('key', 'val')]) >>> from_key_val_list('string') - ValueError: need more than 1 value to unpack + ValueError: cannot encode objects that are not 2-tuples >>> from_key_val_list({'key': 'val'}) OrderedDict([('key', 'val')]) diff --git a/libs/subliminal_patch/core.py b/libs/subliminal_patch/core.py index df38b4e09..0905ce1df 100644 --- a/libs/subliminal_patch/core.py +++ b/libs/subliminal_patch/core.py @@ -396,7 +396,12 @@ class SZProviderPool(ProviderPool): if not subtitle.hash_verifiable and "hash" in matches: can_verify_series = False - if can_verify_series and not {"series", "season", "episode"}.issubset(orig_matches): + matches_series = False + if {"season", "episode"}.issubset(orig_matches) and \ + ("series" in orig_matches or "imdb_id" in orig_matches): + matches_series = True + + if can_verify_series and not matches_series: logger.debug("%r: Skipping subtitle with score %d, because it doesn't match our series/episode", subtitle, score) continue diff --git a/libs/subliminal_patch/http.py b/libs/subliminal_patch/http.py index 84fa3f2c4..a7292ff52 100644 --- a/libs/subliminal_patch/http.py +++ b/libs/subliminal_patch/http.py @@ -1,4 +1,6 @@ # coding=utf-8 +import json +from collections import OrderedDict import certifi import ssl @@ -9,9 +11,7 @@ import requests import xmlrpclib import dns.resolver -from collections import OrderedDict from requests import exceptions -from requests.utils import default_user_agent from urllib3.util import connection from retry.api import retry_call from exceptions import APIThrottled @@ -35,12 +35,6 @@ except AttributeError: default_ssl_context = None -custom_resolver = dns.resolver.Resolver(configure=False) - -# 8.8.8.8 is Google's public DNS server -custom_resolver.nameservers = ['8.8.8.8', '1.1.1.1'] - - class TimeoutSession(requests.Session): timeout = 10 @@ -61,54 +55,65 @@ class CertifiSession(TimeoutSession): self.verify = pem_file -class CFSession(CloudflareScraper, CertifiSession, TimeoutSession): +class CFSession(CloudflareScraper): def __init__(self): super(CFSession, self).__init__() - self.headers = OrderedDict([ - ('User-Agent', default_user_agent()), - ('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'), - ('Accept-Language', 'en-US,en;q=0.5'), - ('Accept-Encoding', 'gzip, deflate'), - ('Connection', 'keep-alive'), - ('Pragma', 'no-cache'), - ('Cache-Control', 'no-cache'), - ('Upgrade-Insecure-Requests', '1'), - ('DNT', '1'), - ]) self.debug = os.environ.get("CF_DEBUG", False) def request(self, method, url, *args, **kwargs): parsed_url = urlparse(url) domain = parsed_url.netloc - cache_key = "cf_data_%s" % domain + cache_key = "cf_data2_%s" % domain - if not self.cookies.get("__cfduid", "", domain=domain): + if not self.cookies.get("cf_clearance", "", domain=domain): cf_data = region.get(cache_key) if cf_data is not NO_VALUE: - cf_cookies, user_agent = cf_data + cf_cookies, user_agent, hdrs = cf_data logger.debug("Trying to use old cf data for %s: %s", domain, cf_data) for cookie, value in cf_cookies.iteritems(): self.cookies.set(cookie, value, domain=domain) - self.headers['User-Agent'] = user_agent + self._hdrs = hdrs + self._ua = user_agent + self.headers['User-Agent'] = self._ua ret = super(CFSession, self).request(method, url, *args, **kwargs) - try: - cf_data = self.get_live_tokens(domain) - except: - pass - else: - if cf_data != region.get(cache_key) and self.cookies.get("__cfduid", "", domain=domain)\ - and self.cookies.get("cf_clearance", "", domain=domain): - logger.debug("Storing cf data for %s: %s", domain, cf_data) - region.set(cache_key, cf_data) + if self._was_cf: + self._was_cf = False + logger.debug("We've hit CF, trying to store previous data") + try: + cf_data = self.get_cf_live_tokens(domain) + except: + logger.debug("Couldn't get CF live tokens for re-use. Cookies: %r", self.cookies) + pass + else: + if cf_data != region.get(cache_key) and cf_data[0]["cf_clearance"]: + logger.debug("Storing cf data for %s: %s", domain, cf_data) + region.set(cache_key, cf_data) return ret + def get_cf_live_tokens(self, domain): + for d in self.cookies.list_domains(): + if d.startswith(".") and d in ("." + domain): + cookie_domain = d + break + else: + raise ValueError( + "Unable to find Cloudflare cookies. Does the site actually have " + "Cloudflare IUAM (\"I'm Under Attack Mode\") enabled?") -class RetryingSession(CFSession): + return (OrderedDict(filter(lambda x: x[1], [ + ("__cfduid", self.cookies.get("__cfduid", "", domain=cookie_domain)), + ("cf_clearance", self.cookies.get("cf_clearance", "", domain=cookie_domain)) + ])), + self._ua, self._hdrs + ) + + +class RetryingSession(CertifiSession): proxied_functions = ("get", "post") def __init__(self): @@ -146,6 +151,10 @@ class RetryingSession(CFSession): return self.retry_method("post", *args, **kwargs) +class RetryingCFSession(RetryingSession, CFSession): + pass + + class SubZeroRequestsTransport(xmlrpclib.SafeTransport): """ Drop in Transport for xmlrpclib that uses Requests instead of httplib @@ -216,25 +225,62 @@ _orig_create_connection = connection.create_connection dns_cache = {} -def set_custom_resolver(): +_custom_resolver = None +_custom_resolver_ips = None + + +def patch_create_connection(): + if hasattr(connection.create_connection, "_sz_patched"): + return + def patched_create_connection(address, *args, **kwargs): """Wrap urllib3's create_connection to resolve the name elsewhere""" # resolve hostname to an ip address; use your own # resolver here, as otherwise the system resolver will be used. + global _custom_resolver, _custom_resolver_ips, dns_cache host, port = address - if host in dns_cache: - ip = dns_cache[host] - logger.debug("Using %s=%s from cache", host, ip) - else: - try: - ip = custom_resolver.query(host)[0].address - dns_cache[host] = ip - except dns.exception.DNSException: - logger.warning("Couldn't resolve %s with DNS: %s", host, custom_resolver.nameservers) - return _orig_create_connection((host, port), *args, **kwargs) - logger.debug("Resolved %s to %s using %s", host, ip, custom_resolver.nameservers) + __custom_resolver_ips = os.environ.get("dns_resolvers", None) - return _orig_create_connection((ip, port), *args, **kwargs) + # resolver ips changed in the meantime? + if __custom_resolver_ips != _custom_resolver_ips: + _custom_resolver = None + _custom_resolver_ips = __custom_resolver_ips + dns_cache = {} + custom_resolver = _custom_resolver + + if not custom_resolver: + if _custom_resolver_ips: + logger.debug("DNS: Trying to use custom DNS resolvers: %s", _custom_resolver_ips) + custom_resolver = dns.resolver.Resolver(configure=False) + custom_resolver.lifetime = 8.0 + try: + custom_resolver.nameservers = json.loads(_custom_resolver_ips) + except: + logger.debug("DNS: Couldn't load custom DNS resolvers: %s", _custom_resolver_ips) + else: + _custom_resolver = custom_resolver + + if custom_resolver: + if host in dns_cache: + ip = dns_cache[host] + logger.debug("DNS: Using %s=%s from cache", host, ip) + return _orig_create_connection((ip, port), *args, **kwargs) + else: + try: + ip = custom_resolver.query(host)[0].address + logger.debug("DNS: Resolved %s to %s using %s", host, ip, custom_resolver.nameservers) + dns_cache[host] = ip + except dns.exception.DNSException: + logger.warning("DNS: Couldn't resolve %s with DNS: %s", host, custom_resolver.nameservers) + raise + + return _orig_create_connection((host, port), *args, **kwargs) + + patch_create_connection._sz_patched = True connection.create_connection = patched_create_connection + + +patch_create_connection() + diff --git a/libs/subliminal_patch/providers/opensubtitles.py b/libs/subliminal_patch/providers/opensubtitles.py index 4ce3aacea..a314087c2 100644 --- a/libs/subliminal_patch/providers/opensubtitles.py +++ b/libs/subliminal_patch/providers/opensubtitles.py @@ -12,10 +12,11 @@ from dogpile.cache.api import NO_VALUE from subliminal.exceptions import ConfigurationError, ServiceUnavailable from subliminal.providers.opensubtitles import OpenSubtitlesProvider as _OpenSubtitlesProvider,\ OpenSubtitlesSubtitle as _OpenSubtitlesSubtitle, Episode, Movie, ServerProxy, Unauthorized, NoSession, \ - DownloadLimitReached, InvalidImdbid, UnknownUserAgent, DisabledUserAgent, OpenSubtitlesError, sanitize + DownloadLimitReached, InvalidImdbid, UnknownUserAgent, DisabledUserAgent, OpenSubtitlesError from mixins import ProviderRetryMixin from subliminal.subtitle import fix_line_ending from subliminal_patch.http import SubZeroRequestsTransport +from subliminal_patch.utils import sanitize from subliminal.cache import region from subliminal_patch.score import framerate_equal from subzero.language import Language @@ -173,7 +174,7 @@ class OpenSubtitlesProvider(ProviderRetryMixin, _OpenSubtitlesProvider): logger.info('Logging in') - token = region.get("os_token", expiration_time=3600) + token = region.get("os_token") if token is not NO_VALUE: try: logger.debug('Trying previous token: %r', token[:10]+"X"*(len(token)-10)) @@ -181,7 +182,7 @@ class OpenSubtitlesProvider(ProviderRetryMixin, _OpenSubtitlesProvider): self.token = token logger.debug("Using previous login token: %r", token[:10]+"X"*(len(token)-10)) return - except: + except (NoSession, Unauthorized): logger.debug('Token not valid.') pass diff --git a/libs/subliminal_patch/providers/subscene.py b/libs/subliminal_patch/providers/subscene.py index 1e2a2a2c8..0025470cf 100644 --- a/libs/subliminal_patch/providers/subscene.py +++ b/libs/subliminal_patch/providers/subscene.py @@ -12,11 +12,11 @@ from zipfile import ZipFile from babelfish import language_converters from guessit import guessit -from requests import Session from dogpile.cache.api import NO_VALUE from subliminal import Episode, ProviderError from subliminal.cache import region from subliminal.utils import sanitize_release_group +from subliminal_patch.http import RetryingCFSession from subliminal_patch.providers import Provider from subliminal_patch.providers.mixins import ProviderSubtitleArchiveMixin from subliminal_patch.subtitle import Subtitle, guess_matches @@ -125,9 +125,7 @@ class SubsceneProvider(Provider, ProviderSubtitleArchiveMixin): def initialize(self): logger.info("Creating session") - self.session = Session() - from .utils import FIRST_THOUSAND_OR_SO_USER_AGENTS as AGENT_LIST - self.session.headers['User-Agent'] = AGENT_LIST[randint(0, len(AGENT_LIST) - 1)] + self.session = RetryingCFSession() def terminate(self): logger.info("Closing session") diff --git a/libs/subliminal_patch/providers/titlovi.py b/libs/subliminal_patch/providers/titlovi.py index 860932ca5..b076680e9 100644 --- a/libs/subliminal_patch/providers/titlovi.py +++ b/libs/subliminal_patch/providers/titlovi.py @@ -12,8 +12,9 @@ from bs4 import BeautifulSoup from zipfile import ZipFile, is_zipfile from rarfile import RarFile, is_rarfile from babelfish import language_converters, Script -from requests import Session, RequestException +from requests import RequestException from guessit import guessit +from subliminal_patch.http import RetryingCFSession from subliminal_patch.providers import Provider from subliminal_patch.providers.mixins import ProviderSubtitleArchiveMixin from subliminal_patch.subtitle import Subtitle @@ -138,13 +139,8 @@ class TitloviProvider(Provider, ProviderSubtitleArchiveMixin): download_url = server_url + '/download/?type=1&mediaid=' def initialize(self): - self.session = Session() - logger.debug("Using random user agents") - self.session.headers['User-Agent'] = AGENT_LIST[randint(0, len(AGENT_LIST) - 1)] - logger.debug('User-Agent set to %s', self.session.headers['User-Agent']) - self.session.headers['Referer'] = self.server_url - logger.debug('Referer set to %s', self.session.headers['Referer']) - load_verification("titlovi", self.session) + self.session = RetryingCFSession() + #load_verification("titlovi", self.session) def terminate(self): self.session.close() @@ -185,42 +181,8 @@ class TitloviProvider(Provider, ProviderSubtitleArchiveMixin): r = self.session.get(self.search_url, params=params, timeout=10) r.raise_for_status() except RequestException as e: - captcha_passed = False - if e.response.status_code == 403 and "data-sitekey" in e.response.content: - logger.info('titlovi: Solving captcha. This might take a couple of minutes, but should only ' - 'happen once every so often') - - site_key = re.search(r'data-sitekey="(.+?)"', e.response.content).group(1) - challenge_s = re.search(r'type="hidden" name="s" value="(.+?)"', e.response.content).group(1) - challenge_ray = re.search(r'data-ray="(.+?)"', e.response.content).group(1) - if not all([site_key, challenge_s, challenge_ray]): - raise Exception("titlovi: Captcha site-key not found!") - - pitcher = pitchers.get_pitcher()("titlovi", e.request.url, site_key, - user_agent=self.session.headers["User-Agent"], - cookies=self.session.cookies.get_dict(), - is_invisible=True) - - result = pitcher.throw() - if not result: - raise Exception("titlovi: Couldn't solve captcha!") - - s_params = { - "s": challenge_s, - "id": challenge_ray, - "g-recaptcha-response": result, - } - r = self.session.get(self.server_url + "/cdn-cgi/l/chk_captcha", params=s_params, timeout=10, - allow_redirects=False) - r.raise_for_status() - r = self.session.get(self.search_url, params=params, timeout=10) - r.raise_for_status() - store_verification("titlovi", self.session) - captcha_passed = True - - if not captcha_passed: - logger.exception('RequestException %s', e) - break + logger.exception('RequestException %s', e) + break else: try: soup = BeautifulSoup(r.content, 'lxml') diff --git a/libs/subliminal_patch/subtitle.py b/libs/subliminal_patch/subtitle.py index 69a3c1e5b..daa922359 100644 --- a/libs/subliminal_patch/subtitle.py +++ b/libs/subliminal_patch/subtitle.py @@ -16,7 +16,8 @@ from pysubs2.subrip import parse_tags, MAX_REPRESENTABLE_TIME from pysubs2.time import ms_to_times from subzero.modification import SubtitleModifications from subliminal import Subtitle as Subtitle_ -from subliminal.subtitle import Episode, Movie, sanitize_release_group, sanitize, get_equivalent_release_groups +from subliminal.subtitle import Episode, Movie, sanitize_release_group, get_equivalent_release_groups +from subliminal_patch.utils import sanitize from ftfy import fix_text logger = logging.getLogger(__name__) @@ -358,9 +359,14 @@ def guess_matches(video, guess, partial=False): matches = set() if isinstance(video, Episode): # series - if video.series and 'title' in guess and sanitize(guess['title']) in ( - sanitize(name) for name in [video.series] + video.alternative_series): - matches.add('series') + if video.series and 'title' in guess: + titles = guess["title"] + if not isinstance(titles, types.ListType): + titles = [titles] + + for title in titles: + if sanitize(title) in (sanitize(name) for name in [video.series] + video.alternative_series): + matches.add('series') # title if video.title and 'episode_title' in guess and sanitize(guess['episode_title']) == sanitize(video.title): matches.add('title') diff --git a/libs/subzero/constants.py b/libs/subzero/constants.py index 43297b2a2..b89da8199 100644 --- a/libs/subzero/constants.py +++ b/libs/subzero/constants.py @@ -2,7 +2,7 @@ OS_PLEX_USERAGENT = 'plexapp.com v9.0' -DEPENDENCY_MODULE_NAMES = ['subliminal', 'subliminal_patch', 'enzyme', 'guessit', 'subzero', 'libfilebot'] +DEPENDENCY_MODULE_NAMES = ['subliminal', 'subliminal_patch', 'enzyme', 'guessit', 'subzero', 'libfilebot', 'cfscrape'] PERSONAL_MEDIA_IDENTIFIER = "com.plexapp.agents.none" PLUGIN_IDENTIFIER_SHORT = "subzero" PLUGIN_IDENTIFIER = "com.plexapp.agents.%s" % PLUGIN_IDENTIFIER_SHORT diff --git a/libs/urllib3/__init__.py b/libs/urllib3/__init__.py index 75725167e..61915465f 100644 --- a/libs/urllib3/__init__.py +++ b/libs/urllib3/__init__.py @@ -27,7 +27,7 @@ from logging import NullHandler __author__ = 'Andrey Petrov (andrey.petrov@shazow.net)' __license__ = 'MIT' -__version__ = '1.24' +__version__ = '1.24.2' __all__ = ( 'HTTPConnectionPool', diff --git a/libs/urllib3/contrib/pyopenssl.py b/libs/urllib3/contrib/pyopenssl.py index 7c0e9465d..5ab7803ac 100644 --- a/libs/urllib3/contrib/pyopenssl.py +++ b/libs/urllib3/contrib/pyopenssl.py @@ -184,6 +184,9 @@ def _dnsname_to_stdlib(name): except idna.core.IDNAError: return None + if ':' in name: + return name + name = idna_encode(name) if name is None: return None diff --git a/libs/urllib3/poolmanager.py b/libs/urllib3/poolmanager.py index fe5491cfd..32bd97302 100644 --- a/libs/urllib3/poolmanager.py +++ b/libs/urllib3/poolmanager.py @@ -7,6 +7,7 @@ from ._collections import RecentlyUsedContainer from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool from .connectionpool import port_by_scheme from .exceptions import LocationValueError, MaxRetryError, ProxySchemeUnknown +from .packages import six from .packages.six.moves.urllib.parse import urljoin from .request import RequestMethods from .util.url import parse_url @@ -342,8 +343,10 @@ class PoolManager(RequestMethods): # conn.is_same_host() which may use socket.gethostbyname() in the future. if (retries.remove_headers_on_redirect and not conn.is_same_host(redirect_location)): - for header in retries.remove_headers_on_redirect: - kw['headers'].pop(header, None) + headers = list(six.iterkeys(kw['headers'])) + for header in headers: + if header.lower() in retries.remove_headers_on_redirect: + kw['headers'].pop(header, None) try: retries = retries.increment(method, url, response=response, _pool=conn) diff --git a/libs/urllib3/response.py b/libs/urllib3/response.py index f0cfbb549..c112690b0 100644 --- a/libs/urllib3/response.py +++ b/libs/urllib3/response.py @@ -69,9 +69,9 @@ class GzipDecoder(object): return getattr(self._obj, name) def decompress(self, data): - ret = b'' + ret = bytearray() if self._state == GzipDecoderState.SWALLOW_DATA or not data: - return ret + return bytes(ret) while True: try: ret += self._obj.decompress(data) @@ -81,11 +81,11 @@ class GzipDecoder(object): self._state = GzipDecoderState.SWALLOW_DATA if previous_state == GzipDecoderState.OTHER_MEMBERS: # Allow trailing garbage acceptable in other gzip clients - return ret + return bytes(ret) raise data = self._obj.unused_data if not data: - return ret + return bytes(ret) self._state = GzipDecoderState.OTHER_MEMBERS self._obj = zlib.decompressobj(16 + zlib.MAX_WBITS) diff --git a/libs/urllib3/util/retry.py b/libs/urllib3/util/retry.py index e7d0abd61..02429ee8e 100644 --- a/libs/urllib3/util/retry.py +++ b/libs/urllib3/util/retry.py @@ -179,7 +179,8 @@ class Retry(object): self.raise_on_status = raise_on_status self.history = history or tuple() self.respect_retry_after_header = respect_retry_after_header - self.remove_headers_on_redirect = remove_headers_on_redirect + self.remove_headers_on_redirect = frozenset([ + h.lower() for h in remove_headers_on_redirect]) def new(self, **kw): params = dict( diff --git a/libs/urllib3/util/ssl_.py b/libs/urllib3/util/ssl_.py index 24ee26d63..5ae435827 100644 --- a/libs/urllib3/util/ssl_.py +++ b/libs/urllib3/util/ssl_.py @@ -263,6 +263,8 @@ def create_urllib3_context(ssl_version=None, cert_reqs=None, """ context = SSLContext(ssl_version or ssl.PROTOCOL_SSLv23) + context.set_ciphers(ciphers or DEFAULT_CIPHERS) + # Setting the default here, as we may have no ssl module on import cert_reqs = ssl.CERT_REQUIRED if cert_reqs is None else cert_reqs @@ -325,7 +327,10 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, if e.errno == errno.ENOENT: raise SSLError(e) raise - elif getattr(context, 'load_default_certs', None) is not None: + + # Don't load system certs unless there were no CA certs or + # SSLContext object specified manually. + elif ssl_context is None and hasattr(context, 'load_default_certs'): # try to load OS default certs; works well on Windows (require Python3.4+) context.load_default_certs() From efcf5161ad1b62deb37e4fe3df13d23529b913b7 Mon Sep 17 00:00:00 2001 From: Halali Date: Tue, 30 Apr 2019 20:52:18 +0200 Subject: [PATCH 02/23] Fix for movies history paging --- views/historymovies.tpl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/views/historymovies.tpl b/views/historymovies.tpl index 5f4e0d041..3c3cc37a3 100644 --- a/views/historymovies.tpl +++ b/views/historymovies.tpl @@ -205,15 +205,15 @@ }); $('.fast.backward').on('click', function(){ - loadURLseries(1); + loadURLmovies(1); }); $('.backward:not(.fast)').on('click', function(){ - loadURLseries({{int(page)-1}}); + loadURLmovies({{int(page)-1}}); }); $('.forward:not(.fast)').on('click', function(){ - loadURLseries({{int(page)+1}}); + loadURLmovies({{int(page)+1}}); }); $('.fast.forward').on('click', function(){ - loadURLseries({{int(max_page)}}); + loadURLmovies({{int(max_page)}}); }); \ No newline at end of file From 164230c7cd006c3010a7d12c3f52011f68dd2ea9 Mon Sep 17 00:00:00 2001 From: morpheus65535 Date: Wed, 1 May 2019 06:31:15 -0400 Subject: [PATCH 03/23] Fix input type for DBC password. --- views/settings.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/views/settings.tpl b/views/settings.tpl index 405106f39..3111f8d4d 100644 --- a/views/settings.tpl +++ b/views/settings.tpl @@ -1312,7 +1312,7 @@
+ type="password" value="{{ settings.deathbycaptcha.password }}">
From d910db796531f0e7169e9ff41c2f316db8d1d4a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=A2irts=20Kokars?= Date: Thu, 2 May 2019 21:32:52 +0300 Subject: [PATCH 04/23] add subtitri.nekur.net and subtitri.id.lv subtitle providers --- libs/subliminal_patch/providers/nekur.py | 178 ++++++++++++++++++ libs/subliminal_patch/providers/subtitriid.py | 163 ++++++++++++++++ views/settings.tpl | 44 +++++ 3 files changed, 385 insertions(+) create mode 100644 libs/subliminal_patch/providers/nekur.py create mode 100644 libs/subliminal_patch/providers/subtitriid.py diff --git a/libs/subliminal_patch/providers/nekur.py b/libs/subliminal_patch/providers/nekur.py new file mode 100644 index 000000000..859025865 --- /dev/null +++ b/libs/subliminal_patch/providers/nekur.py @@ -0,0 +1,178 @@ +# -*- coding: utf-8 -*- +import io +import logging +from random import randint + +from zipfile import ZipFile, is_zipfile +from rarfile import RarFile, is_rarfile + +from guessit import guessit +from requests import Session +from bs4 import NavigableString +from ftfy import fix_text +from subzero.language import Language + +from subliminal_patch.providers import Provider +from subliminal_patch.providers.mixins import ProviderSubtitleArchiveMixin +from subliminal_patch.subtitle import Subtitle +from subliminal_patch.score import framerate_equal +from subliminal.exceptions import ProviderError +from subliminal.providers import ParserBeautifulSoup +from subliminal.subtitle import sanitize, guess_matches +from subliminal.video import Movie +from .utils import FIRST_THOUSAND_OR_SO_USER_AGENTS as AGENT_LIST + +logger = logging.getLogger(__name__) + + +class NekurSubtitle(Subtitle): + """Nekur Subtitle.""" + provider_name = 'nekur' + + def __init__(self, language, page_link, download_link, title, year, imdb_id, fps, notes): + super(NekurSubtitle, self).__init__(language, page_link=page_link) + self.download_link = download_link + self.title = title + self.year = year + self.imdb_id = imdb_id + self.fps = fps + self.notes = notes + self.matches = None + # self.encoding = 'utf-16' + + @property + def id(self): + return self.download_link + + def get_matches(self, video): + matches = set() + + if isinstance(video, Movie): + # title + if video.title and sanitize(self.title) == sanitize(video.title): + matches.add('title') + # year + if video.year and self.year == video.year: + matches.add('year') + # imdb id + if video.imdb_id and self.imdb_id == video.imdb_id: + matches.add('imdb_id') + # fps + if video.fps and self.fps and not framerate_equal(video.fps, self.fps): + logger.warning("nekur: Wrong FPS (expected: %s, got: %s)", video.fps, self.fps) + # guess additional info from notes + matches |= guess_matches(video, guessit(self.notes, {'type': 'movie'}), partial=True) + + self.matches = matches + return matches + + +class NekurProvider(Provider, ProviderSubtitleArchiveMixin): + """Nekur Provider.""" + subtitle_class = NekurSubtitle + languages = {Language('lva', 'LV')} | {Language.fromalpha2(l) for l in ['lv']} + server_url = 'http://subtitri.nekur.net/' + search_url = server_url + 'modules/Subtitles.php' + + def __init__(self): + self.session = None + + def initialize(self): + self.session = Session() + self.session.headers['User-Agent'] = AGENT_LIST[randint(0, len(AGENT_LIST) - 1)] + self.session.headers['Referer'] = self.server_url + + def terminate(self): + self.session.close() + + def query(self, title): + subtitles = [] + + data = { + 'ajax': '1', + 'sSearch': title, + } + + r = self.session.post(self.search_url, data=data, timeout=10) + r.raise_for_status() + + if not r.content: + logger.debug('No data returned from provider') + return [] + + soup = ParserBeautifulSoup(r.content.decode('utf-8', 'ignore'), ['lxml', 'html.parser']) + + # loop over subtitle cells + rows = soup.select('tbody > tr') + for row in rows: + # title + title_anchor_el = row.select_one('.title > a') + title_inner_text = [element for element in title_anchor_el if isinstance(element, NavigableString)] + title = title_inner_text[0].strip() + + # year + year = row.select_one('.year').text.strip('()') + + # download link + href = title_anchor_el.get('href') + download_link = self.server_url + href + + # imdb id + imdb_td = row.select_one('td:nth-of-type(4)') + imdb_link = imdb_td.select_one('a').get('href') + imdb_id = imdb_link.split('/')[-2] + + # fps + fps = row.select_one('.fps').text.strip() + + # additional notes + notes = row.select_one('.notes').text.strip() + + # page link = archive link (there is no seperate subtitle page link) + page_link = 'http://subtitri.nekur.net/filmu-subtitri/' + + # create/add the subitle + subtitle = self.subtitle_class(Language.fromalpha2('lv'), page_link, download_link, title, year, imdb_id, fps, notes) + logger.debug('nekur: Found subtitle %r', subtitle) + subtitles.append(subtitle) + + return subtitles + + def list_subtitles(self, video, languages): + if isinstance(video, Movie): + titles = [video.title] + video.alternative_titles + else: + titles = [] + + subtitles = [] + # query for subtitles + for title in titles: + if isinstance(video, Movie): + subtitles += [s for s in self.query(title) if s.language in languages] + + return subtitles + + def download_subtitle(self, subtitle): + if isinstance(subtitle, NekurSubtitle): + # download the subtitle + r = self.session.get(subtitle.download_link, timeout=10) + r.raise_for_status() + + # open the archive + archive_stream = io.BytesIO(r.content) + if is_rarfile(archive_stream): + archive = RarFile(archive_stream) + elif is_zipfile(archive_stream): + archive = ZipFile(archive_stream) + else: + subtitle.content = r.content + if subtitle.is_valid(): + return + subtitle.content = None + + raise ProviderError('Unidentified archive type') + + subtitle_content = self.get_subtitle_from_archive(subtitle, archive) + # fix content encoding (utf-16 encoded by default) + fixed_subtitle_content = fix_text(subtitle_content.decode('utf-16'), {'uncurl_quotes': False, 'fix_character_width': False}).encode(encoding='utf-8') + subtitle.content = fixed_subtitle_content diff --git a/libs/subliminal_patch/providers/subtitriid.py b/libs/subliminal_patch/providers/subtitriid.py new file mode 100644 index 000000000..4a3259fa3 --- /dev/null +++ b/libs/subliminal_patch/providers/subtitriid.py @@ -0,0 +1,163 @@ +# -*- coding: utf-8 -*- +import io +import logging +from random import randint + +from zipfile import ZipFile, is_zipfile +from rarfile import RarFile, is_rarfile + +from requests import Session +from ftfy import fix_text +from subzero.language import Language + +from subliminal_patch.providers import Provider +from subliminal_patch.providers.mixins import ProviderSubtitleArchiveMixin +from subliminal_patch.subtitle import Subtitle +from subliminal.exceptions import ProviderError +from subliminal.providers import ParserBeautifulSoup +from subliminal.subtitle import sanitize +from subliminal.video import Movie +from .utils import FIRST_THOUSAND_OR_SO_USER_AGENTS as AGENT_LIST + +logger = logging.getLogger(__name__) + + +class SubtitriIdSubtitle(Subtitle): + """subtitri.id.lv Subtitle.""" + provider_name = 'subtitriid' + + def __init__(self, language, page_link, download_link, title, year, imdb_id): + super(SubtitriIdSubtitle, self).__init__(language, page_link=page_link) + self.download_link = download_link + self.title = title + self.year = year + self.imdb_id = imdb_id + self.matches = None + # self.encoding = 'utf-16' + + @property + def id(self): + return self.download_link + + def get_matches(self, video): + matches = set() + if isinstance(video, Movie): + # title + if video.title and sanitize(self.title) == sanitize(video.title): + matches.add('title') + # year + if video.year and self.year == video.year: + matches.add('year') + # imdb id + if video.imdb_id and self.imdb_id == video.imdb_id: + matches.add('imdb_id') + + self.matches = matches + return matches + + +class SubtitriIdProvider(Provider, ProviderSubtitleArchiveMixin): + """subtitri.id.lv Provider.""" + subtitle_class = SubtitriIdSubtitle + languages = {Language('lva', 'LV')} | {Language.fromalpha2(l) for l in ['lv']} + server_url = 'http://subtitri.id.lv' + search_url = server_url + '/search/' + + def __init__(self): + self.session = None + + def initialize(self): + self.session = Session() + self.session.headers['User-Agent'] = AGENT_LIST[randint(0, len(AGENT_LIST) - 1)] + self.session.headers['Referer'] = self.server_url + + def terminate(self): + self.session.close() + + def query(self, title): + subtitles = [] + + r = self.session.get(self.search_url, params = {'q': title}, timeout=10) + r.raise_for_status() + + if not r.content: + logger.debug('No data returned from provider') + return [] + + soup = ParserBeautifulSoup(r.content.decode('utf-8', 'ignore'), ['lxml', 'html.parser']) + + # loop over subtitle cells + rows = soup.select('.eBlock') + for row in rows: + result_anchor_el = row.select_one('.eTitle > a') + + # page link + page_link = result_anchor_el.get('href') + + # fetch/parse additional info + r = self.session.get(page_link, timeout=10) + soup = ParserBeautifulSoup(r.content.decode('utf-8', 'ignore'), ['lxml', 'html.parser']) + + # title + movie_titles_string = soup.select_one('.main-header').text.strip() + movie_titles_list = movie_titles_string.split(' / ') + title = movie_titles_list[-1] + # # TODO alternate titles(?) + # alternate_titles = movie_title_list.remove(title) + + # year + year = soup.select_one('#film-page-year').text.strip() + + # imdb id + imdb_link = soup.select_one('#actors-page > a').get('href') + imdb_id = imdb_link.split('/')[-2] + + # download link + href = soup.select_one('.hvr').get('href') + download_link = self.server_url + href + + # create/add the subitle + subtitle = self.subtitle_class(Language.fromalpha2('lv'), page_link, download_link, title, year, imdb_id) + logger.debug('subtitri.id.lv: Found subtitle %r', subtitle) + subtitles.append(subtitle) + + return subtitles + + def list_subtitles(self, video, languages): + if isinstance(video, Movie): + titles = [video.title] + video.alternative_titles + else: + titles = [] + + subtitles = [] + # query for subtitles + for title in titles: + if isinstance(video, Movie): + subtitles += [s for s in self.query(title) if s.language in languages] + + return subtitles + + def download_subtitle(self, subtitle): + if isinstance(subtitle, SubtitriIdSubtitle): + # download the subtitle + r = self.session.get(subtitle.download_link, timeout=10) + r.raise_for_status() + + # open the archive + archive_stream = io.BytesIO(r.content) + if is_rarfile(archive_stream): + archive = RarFile(archive_stream) + elif is_zipfile(archive_stream): + archive = ZipFile(archive_stream) + else: + subtitle.content = r.content + if subtitle.is_valid(): + return + subtitle.content = None + + raise ProviderError('Unidentified archive type') + + subtitle_content = self.get_subtitle_from_archive(subtitle, archive) + # fix content encoding (utf-16 encoded by default) + fixed_subtitle_content = fix_text(subtitle_content.decode('utf-16'), {'uncurl_quotes': False, 'fix_character_width': False}).encode(encoding='utf-8') + subtitle.content = fixed_subtitle_content diff --git a/views/settings.tpl b/views/settings.tpl index 3111f8d4d..bf0f49616 100644 --- a/views/settings.tpl +++ b/views/settings.tpl @@ -1472,6 +1472,28 @@ +
+
+ +
+
+
+ + +
+
+ +
+
+ +
+
@@ -1756,6 +1778,28 @@
+
+
+ +
+
+
+ + +
+
+ +
+
+ +
+
From aba93482b047d62e59a287c75be78d1563f0f7ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Louis=20V=C3=A9zina?= <5130500+morpheus65535@users.noreply.github.com> Date: Fri, 3 May 2019 06:30:46 -0400 Subject: [PATCH 05/23] Some small UI fixes. --- views/movies.tpl | 9 +++------ views/series.tpl | 2 +- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/views/movies.tpl b/views/movies.tpl index 335edc151..fa351e2fd 100644 --- a/views/movies.tpl +++ b/views/movies.tpl @@ -3,7 +3,6 @@ - @@ -53,16 +52,16 @@
- +
- + - + @@ -224,8 +223,6 @@ sessionStorage.clear(); } - $('table').tablesort(); - $('a, button:not(.cancel)').on('click', function(){ $('#loader').addClass('active'); }); diff --git a/views/series.tpl b/views/series.tpl index 35da06870..117e4b707 100644 --- a/views/series.tpl +++ b/views/series.tpl @@ -75,7 +75,7 @@ %import ast %import os %for row in rows: - +
NameName Path Audio
language		 Subtitles
languages		 Hearing-
impaired
{{row[1]}} %if os.path.isdir(row[2]): From 34495b92636093cb3e06116d5485ac76df00e240 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=A2irts=20Kokars?= Date: Fri, 3 May 2019 14:03:50 +0300 Subject: [PATCH 06/23] add providers to the wizard --- views/wizard.tpl | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/views/wizard.tpl b/views/wizard.tpl index 8976abd73..dd6c9ae1d 100644 --- a/views/wizard.tpl +++ b/views/wizard.tpl @@ -561,6 +561,28 @@ +
+
+ +
+
+
+ + +
+
+ +
+
+ +
+
@@ -845,6 +867,28 @@
+
+
+ +
+
+
+ + +
+
+ +
+
+ +
+
From 9079e2b3dd0523e73a0261098a95cf8c8a22f95f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=A2irts=20Kokars?= Date: Fri, 3 May 2019 14:06:26 +0300 Subject: [PATCH 07/23] add providers to the readme --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 198916444..fccceb4d0 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,8 @@ If you need something that is not already part of Bazarr, feel free to create a * Subsunacs.net * Subs4Free * Subs4Series +* Subtitri.id.lv +* Subtitri.nekur.net * SubZ * Supersubtitles * Titlovi From 1f8469f83a48a012138ce8f99d572dfbecb1a8bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=A2irts=20Kokars?= Date: Fri, 3 May 2019 14:08:39 +0300 Subject: [PATCH 08/23] remove unused code (alternate titles) --- libs/subliminal_patch/providers/subtitriid.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/libs/subliminal_patch/providers/subtitriid.py b/libs/subliminal_patch/providers/subtitriid.py index 4a3259fa3..cec25d37d 100644 --- a/libs/subliminal_patch/providers/subtitriid.py +++ b/libs/subliminal_patch/providers/subtitriid.py @@ -102,8 +102,6 @@ class SubtitriIdProvider(Provider, ProviderSubtitleArchiveMixin): movie_titles_string = soup.select_one('.main-header').text.strip() movie_titles_list = movie_titles_string.split(' / ') title = movie_titles_list[-1] - # # TODO alternate titles(?) - # alternate_titles = movie_title_list.remove(title) # year year = soup.select_one('#film-page-year').text.strip() From 90645838538bb7346ff9cb60eb76302526e5eb92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Louis=20V=C3=A9zina?= <5130500+morpheus65535@users.noreply.github.com> Date: Fri, 3 May 2019 09:34:08 -0400 Subject: [PATCH 09/23] Fix for non UTF8 file system encoding. --- bazarr/embedded_subs_reader.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bazarr/embedded_subs_reader.py b/bazarr/embedded_subs_reader.py index b4961f7d9..0c4a3a100 100644 --- a/bazarr/embedded_subs_reader.py +++ b/bazarr/embedded_subs_reader.py @@ -2,6 +2,7 @@ import enzyme import logging import os import subprocess +import locale from utils import get_binary @@ -18,7 +19,7 @@ class EmbeddedSubsReader: def list_languages(self, file): if self.ffprobe: try: - return subprocess.check_output([self.ffprobe, "-loglevel", "error", "-select_streams", "s", "-show_entries", "stream_tags=language", "-of", "csv=p=0", file], universal_newlines=True, stderr=subprocess.STDOUT).strip().split("\n") + return subprocess.check_output([self.ffprobe, "-loglevel", "error", "-select_streams", "s", "-show_entries", "stream_tags=language", "-of", "csv=p=0", file.encode(locale.getpreferredencoding())], universal_newlines=True, stderr=subprocess.STDOUT).strip().split("\n") except subprocess.CalledProcessError as e: raise FFprobeError(e.output) if os.path.splitext(file)[1] != '.mkv': From f813708f61ab55ea5017c2da26a4ca29e5309936 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Louis=20V=C3=A9zina?= <5130500+morpheus65535@users.noreply.github.com> Date: Fri, 3 May 2019 13:37:29 -0400 Subject: [PATCH 10/23] Fix for #424. --- bazarr/get_subtitle.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bazarr/get_subtitle.py b/bazarr/get_subtitle.py index 06eea4886..0a1605479 100644 --- a/bazarr/get_subtitle.py +++ b/bazarr/get_subtitle.py @@ -105,9 +105,9 @@ def download_subtitle(path, language, hi, providers, providers_auth, sceneName, logging.debug('BAZARR Searching subtitles for this file: ' + path) if hi == "True": - hi = True + hi = "force HI" else: - hi = False + hi = "force non-HI" language_set = set() if not isinstance(language, types.ListType): @@ -253,9 +253,9 @@ def manual_search(path, language, hi, providers, providers_auth, sceneName, titl final_subtitles = [] if hi == "True": - hi = True + hi = "force HI" else: - hi = False + hi = "force non-HI" language_set = set() for lang in ast.literal_eval(language): lang = alpha3_from_alpha2(lang) From 9f80fcfa80a35d4c1727e0a0d10311bbd8e690c0 Mon Sep 17 00:00:00 2001 From: Zandor Smith Date: Sun, 5 May 2019 20:10:35 +0200 Subject: [PATCH 11/23] Fixed grammar in tooltips on the history page. --- views/historymovies.tpl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/views/historymovies.tpl b/views/historymovies.tpl index 3c3cc37a3..2c04867a9 100644 --- a/views/historymovies.tpl +++ b/views/historymovies.tpl @@ -59,19 +59,19 @@
%if row[0] == 0: -
+
%elif row[0] == 1: -
+
%elif row[0] == 2: -
+
%elif row[0] == 3: -
+
%end @@ -216,4 +216,4 @@ $('.fast.forward').on('click', function(){ loadURLmovies({{int(max_page)}}); }); - \ No newline at end of file + From 85be773190c40c5af412e73cf0c5058d3fa9cec6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Louis=20V=C3=A9zina?= <5130500+morpheus65535@users.noreply.github.com> Date: Sun, 5 May 2019 20:21:46 -0400 Subject: [PATCH 12/23] Typo fix in series history. --- views/historyseries.tpl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/views/historyseries.tpl b/views/historyseries.tpl index a4da3dc43..b3f330c25 100644 --- a/views/historyseries.tpl +++ b/views/historyseries.tpl @@ -61,19 +61,19 @@
%if row[0] == 0: -
+
%elif row[0] == 1: -
+
%elif row[0] == 2: -
+
%elif row[0] == 3: -
+
%end From 91e729ac067542632e9a6fc83bf6d54f6efde97e Mon Sep 17 00:00:00 2001 From: sekkr1 Date: Mon, 6 May 2019 07:27:41 +0300 Subject: [PATCH 13/23] added ignore pgs settings to embedded scan --- bazarr/config.py | 1 + bazarr/embedded_subs_reader.py | 10 ++++++++-- bazarr/main.py | 12 ++++++++++++ views/settings.tpl | 25 +++++++++++++++++++++++++ 4 files changed, 46 insertions(+), 2 deletions(-) diff --git a/bazarr/config.py b/bazarr/config.py index 07a8d6965..0c231f593 100644 --- a/bazarr/config.py +++ b/bazarr/config.py @@ -31,6 +31,7 @@ defaults = { 'page_size': '25', 'minimum_score_movie': '70', 'use_embedded_subs': 'True', + 'ignore_pgs_subs': 'True', 'adaptive_searching': 'False', 'enabled_providers': '', 'throtteled_providers': '{}', diff --git a/bazarr/embedded_subs_reader.py b/bazarr/embedded_subs_reader.py index 0c4a3a100..7cd9bbfff 100644 --- a/bazarr/embedded_subs_reader.py +++ b/bazarr/embedded_subs_reader.py @@ -4,6 +4,7 @@ import os import subprocess import locale +from config import settings from utils import get_binary class NotMKVAndNoFFprobe(Exception): @@ -19,13 +20,18 @@ class EmbeddedSubsReader: def list_languages(self, file): if self.ffprobe: try: - return subprocess.check_output([self.ffprobe, "-loglevel", "error", "-select_streams", "s", "-show_entries", "stream_tags=language", "-of", "csv=p=0", file.encode(locale.getpreferredencoding())], universal_newlines=True, stderr=subprocess.STDOUT).strip().split("\n") + if not settings.general.getboolean('ignore_pgs_subs'): + return subprocess.check_output([self.ffprobe, "-loglevel", "error", "-select_streams", "s", "-show_entries", "stream_tags=language", "-of", "csv=p=0", file.encode(locale.getpreferredencoding())], universal_newlines=True, stderr=subprocess.STDOUT).strip().split("\n") + subtitle_tracks = subprocess.check_output([self.ffprobe, "-loglevel", "error", "-select_streams", "s", "-show_entries", "stream=codec_name:stream_tags=language", "-of", "csv=p=0", file.encode(locale.getpreferredencoding())], universal_newlines=True, stderr=subprocess.STDOUT).strip().split("\n") + return [lang for (sub_type, lang) in map(lambda subtitle_track: subtitle_track.split(','), subtitle_tracks) if sub_type != 'hdmv_pgs_subtitle'] except subprocess.CalledProcessError as e: raise FFprobeError(e.output) if os.path.splitext(file)[1] != '.mkv': raise NotMKVAndNoFFprobe() with open(file, 'rb') as f: mkv = enzyme.MKV(f) - return [subtitle_track.language for subtitle_track in mkv.subtitle_tracks] + if not settings.general.getboolean('ignore_pgs_subs'): + return [subtitle_track.language for subtitle_track in mkv.subtitle_tracks] + return [subtitle_track.language for subtitle_track in mkv.subtitle_tracks if subtitle_track.codec_id != "S_HDMV/PGS"] embedded_subs_reader = EmbeddedSubsReader() \ No newline at end of file diff --git a/bazarr/main.py b/bazarr/main.py index 3e794de6c..86f7073ae 100644 --- a/bazarr/main.py +++ b/bazarr/main.py @@ -277,6 +277,11 @@ def save_wizard(): settings_general_embedded = 'False' else: settings_general_embedded = 'True' + settings_general_ignore_pgs = request.forms.get('settings_general_ignore_pgs') + if settings_general_ignore_pgs is None: + settings_general_ignore_pgs = 'False' + else: + settings_general_ignore_pgs = 'True' settings_subfolder = request.forms.get('settings_subfolder') settings_subfolder_custom = request.forms.get('settings_subfolder_custom') settings_upgrade_subs = request.forms.get('settings_upgrade_subs') @@ -302,6 +307,7 @@ def save_wizard(): settings.general.subfolder = text_type(settings_subfolder) settings.general.subfolder_custom = text_type(settings_subfolder_custom) settings.general.use_embedded_subs = text_type(settings_general_embedded) + settings.general.ignore_pgs_subs = text_type(settings_general_ignore_pgs) settings.general.upgrade_subs = text_type(settings_upgrade_subs) settings.general.days_to_upgrade_subs = text_type(settings_days_to_upgrade_subs) settings.general.upgrade_manual = text_type(settings_upgrade_manual) @@ -1234,6 +1240,11 @@ def save_settings(): settings_general_embedded = 'False' else: settings_general_embedded = 'True' + settings_general_ignore_pgs = request.forms.get('settings_general_ignore_pgs') + if settings_general_ignore_pgs is None: + settings_general_ignore_pgs = 'False' + else: + settings_general_ignore_pgs = 'True' settings_general_adaptive_searching = request.forms.get('settings_general_adaptive_searching') if settings_general_adaptive_searching is None: settings_general_adaptive_searching = 'False' @@ -1332,6 +1343,7 @@ def save_settings(): settings.general.minimum_score_movie = text_type(settings_general_minimum_score_movies) settings.general.use_embedded_subs = text_type(settings_general_embedded) + settings.general.ignore_pgs_subs = text_type(settings_general_ignore_pgs) settings.general.adaptive_searching = text_type(settings_general_adaptive_searching) settings.general.multithreading = text_type(settings_general_multithreading) diff --git a/views/settings.tpl b/views/settings.tpl index 3111f8d4d..6059709ab 100644 --- a/views/settings.tpl +++ b/views/settings.tpl @@ -1188,6 +1188,25 @@
+
+
+ +
+
+
+ + +
+
+ +
+
@@ -2232,6 +2251,12 @@ $("#settings_embedded").checkbox('uncheck'); } + if ($('#settings_ignore_pgs').data("ignore") === "True") { + $("#settings_ignore_pgs").checkbox('check'); + } else { + $("#settings_ignore_pgs").checkbox('uncheck'); + } + if ($('#settings_only_monitored_sonarr').data("monitored") === "True") { $("#settings_only_monitored_sonarr").checkbox('check'); } else { From 9327126d73c49288ecf0f6dce71530aaf7228a87 Mon Sep 17 00:00:00 2001 From: sekkr1 Date: Mon, 6 May 2019 09:04:00 +0300 Subject: [PATCH 14/23] handle empty output as split returns [''] on those --- bazarr/embedded_subs_reader.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/bazarr/embedded_subs_reader.py b/bazarr/embedded_subs_reader.py index 7cd9bbfff..a4a659c63 100644 --- a/bazarr/embedded_subs_reader.py +++ b/bazarr/embedded_subs_reader.py @@ -21,8 +21,14 @@ class EmbeddedSubsReader: if self.ffprobe: try: if not settings.general.getboolean('ignore_pgs_subs'): - return subprocess.check_output([self.ffprobe, "-loglevel", "error", "-select_streams", "s", "-show_entries", "stream_tags=language", "-of", "csv=p=0", file.encode(locale.getpreferredencoding())], universal_newlines=True, stderr=subprocess.STDOUT).strip().split("\n") - subtitle_tracks = subprocess.check_output([self.ffprobe, "-loglevel", "error", "-select_streams", "s", "-show_entries", "stream=codec_name:stream_tags=language", "-of", "csv=p=0", file.encode(locale.getpreferredencoding())], universal_newlines=True, stderr=subprocess.STDOUT).strip().split("\n") + subtitle_languages = subprocess.check_output([self.ffprobe, "-loglevel", "error", "-select_streams", "s", "-show_entries", "stream_tags=language", "-of", "csv=p=0", file.encode(locale.getpreferredencoding())], universal_newlines=True, stderr=subprocess.STDOUT).strip() + if not subtitle_languages: + return [] + return subtitle_languages.split('\n') + subtitle_tracks = subprocess.check_output([self.ffprobe, "-loglevel", "error", "-select_streams", "s", "-show_entries", "stream=codec_name:stream_tags=language", "-of", "csv=p=0", file.encode(locale.getpreferredencoding())], universal_newlines=True, stderr=subprocess.STDOUT).strip() + if not subtitle_tracks: + return [] + subtitle_tracks = subtitle_tracks.split('\n') return [lang for (sub_type, lang) in map(lambda subtitle_track: subtitle_track.split(','), subtitle_tracks) if sub_type != 'hdmv_pgs_subtitle'] except subprocess.CalledProcessError as e: raise FFprobeError(e.output) From 885586ba70bc1202d97acb4fb0c9b09d918a22b2 Mon Sep 17 00:00:00 2001 From: morpheus65535 Date: Tue, 7 May 2019 03:26:16 +0300 Subject: [PATCH 15/23] defaults to disabled Co-Authored-By: sekkr1 --- bazarr/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bazarr/config.py b/bazarr/config.py index 0c231f593..2bfdcf2bf 100644 --- a/bazarr/config.py +++ b/bazarr/config.py @@ -31,7 +31,7 @@ defaults = { 'page_size': '25', 'minimum_score_movie': '70', 'use_embedded_subs': 'True', - 'ignore_pgs_subs': 'True', + 'ignore_pgs_subs': 'False', 'adaptive_searching': 'False', 'enabled_providers': '', 'throtteled_providers': '{}', From f3b8f22b4351bac2faecd5d2283b3774ea62a016 Mon Sep 17 00:00:00 2001 From: morpheus65535 Date: Tue, 7 May 2019 03:27:00 +0300 Subject: [PATCH 16/23] rename data attr to ignorepgs in html Co-Authored-By: sekkr1 --- views/settings.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/views/settings.tpl b/views/settings.tpl index 6059709ab..138f94d77 100644 --- a/views/settings.tpl +++ b/views/settings.tpl @@ -1193,7 +1193,7 @@
-
+
From cbda709796970eb4bb5946d425c59273eeec7bfa Mon Sep 17 00:00:00 2001 From: morpheus65535 Date: Tue, 7 May 2019 03:27:11 +0300 Subject: [PATCH 17/23] rename data attr to ignorepgs in jquery Co-Authored-By: sekkr1 --- views/settings.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/views/settings.tpl b/views/settings.tpl index 138f94d77..ae3c73f9e 100644 --- a/views/settings.tpl +++ b/views/settings.tpl @@ -2251,7 +2251,7 @@ $("#settings_embedded").checkbox('uncheck'); } - if ($('#settings_ignore_pgs').data("ignore") === "True") { + if ($('#settings_ignore_pgs').data("ignorepgs") === "True") { $("#settings_ignore_pgs").checkbox('check'); } else { $("#settings_ignore_pgs").checkbox('uncheck'); From be7ea23239b80a3c20df3a566cf262de04c67b8d Mon Sep 17 00:00:00 2001 From: sekkr1 Date: Tue, 7 May 2019 03:28:26 +0300 Subject: [PATCH 18/23] removed from wizard --- bazarr/main.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/bazarr/main.py b/bazarr/main.py index 86f7073ae..f57fc2e8e 100644 --- a/bazarr/main.py +++ b/bazarr/main.py @@ -277,11 +277,6 @@ def save_wizard(): settings_general_embedded = 'False' else: settings_general_embedded = 'True' - settings_general_ignore_pgs = request.forms.get('settings_general_ignore_pgs') - if settings_general_ignore_pgs is None: - settings_general_ignore_pgs = 'False' - else: - settings_general_ignore_pgs = 'True' settings_subfolder = request.forms.get('settings_subfolder') settings_subfolder_custom = request.forms.get('settings_subfolder_custom') settings_upgrade_subs = request.forms.get('settings_upgrade_subs') @@ -307,7 +302,6 @@ def save_wizard(): settings.general.subfolder = text_type(settings_subfolder) settings.general.subfolder_custom = text_type(settings_subfolder_custom) settings.general.use_embedded_subs = text_type(settings_general_embedded) - settings.general.ignore_pgs_subs = text_type(settings_general_ignore_pgs) settings.general.upgrade_subs = text_type(settings_upgrade_subs) settings.general.days_to_upgrade_subs = text_type(settings_days_to_upgrade_subs) settings.general.upgrade_manual = text_type(settings_upgrade_manual) From a8840c2fb60df70f75d6652e330855411b40403c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=A2irts=20Kokars?= Date: Wed, 8 May 2019 15:17:01 +0300 Subject: [PATCH 19/23] override subtitle guess_encoding method to not include language-specific encodings and go straight to chardet as it seems to yield far better results --- libs/subliminal_patch/providers/nekur.py | 43 ++++++++++++++++--- libs/subliminal_patch/providers/subtitriid.py | 42 +++++++++++++++--- 2 files changed, 72 insertions(+), 13 deletions(-) diff --git a/libs/subliminal_patch/providers/nekur.py b/libs/subliminal_patch/providers/nekur.py index 859025865..f8950d1f2 100644 --- a/libs/subliminal_patch/providers/nekur.py +++ b/libs/subliminal_patch/providers/nekur.py @@ -8,8 +8,8 @@ from rarfile import RarFile, is_rarfile from guessit import guessit from requests import Session -from bs4 import NavigableString -from ftfy import fix_text +import chardet +from bs4 import NavigableString, UnicodeDammit from subzero.language import Language from subliminal_patch.providers import Provider @@ -38,7 +38,6 @@ class NekurSubtitle(Subtitle): self.fps = fps self.notes = notes self.matches = None - # self.encoding = 'utf-16' @property def id(self): @@ -66,6 +65,39 @@ class NekurSubtitle(Subtitle): self.matches = matches return matches + def guess_encoding(self): + # override default subtitle guess_encoding method to not include language-specific encodings guessing + # chardet encoding detection seem to yield better results + """Guess encoding using chardet. + + :return: the guessed encoding. + :rtype: str + + """ + if self._guessed_encoding: + return self._guessed_encoding + + logger.info('Guessing encoding for language %s', self.language) + + # guess/detect encoding using chardet + encoding = chardet.detect(self.content)['encoding'] + logger.info('Chardet found encoding %s', encoding) + + if not encoding: + # fallback on bs4 + logger.info('Falling back to bs4 detection') + a = UnicodeDammit(self.content) + + logger.info("bs4 detected encoding: %s", a.original_encoding) + + if a.original_encoding: + self._guessed_encoding = a.original_encoding + return a.original_encoding + raise ValueError(u"Couldn't guess the proper encoding for %s", self) + + self._guessed_encoding = encoding + return encoding + class NekurProvider(Provider, ProviderSubtitleArchiveMixin): """Nekur Provider.""" @@ -172,7 +204,4 @@ class NekurProvider(Provider, ProviderSubtitleArchiveMixin): raise ProviderError('Unidentified archive type') - subtitle_content = self.get_subtitle_from_archive(subtitle, archive) - # fix content encoding (utf-16 encoded by default) - fixed_subtitle_content = fix_text(subtitle_content.decode('utf-16'), {'uncurl_quotes': False, 'fix_character_width': False}).encode(encoding='utf-8') - subtitle.content = fixed_subtitle_content + subtitle.content = self.get_subtitle_from_archive(subtitle, archive) diff --git a/libs/subliminal_patch/providers/subtitriid.py b/libs/subliminal_patch/providers/subtitriid.py index cec25d37d..d21ec7324 100644 --- a/libs/subliminal_patch/providers/subtitriid.py +++ b/libs/subliminal_patch/providers/subtitriid.py @@ -7,7 +7,8 @@ from zipfile import ZipFile, is_zipfile from rarfile import RarFile, is_rarfile from requests import Session -from ftfy import fix_text +import chardet +from bs4 import UnicodeDammit from subzero.language import Language from subliminal_patch.providers import Provider @@ -33,7 +34,6 @@ class SubtitriIdSubtitle(Subtitle): self.year = year self.imdb_id = imdb_id self.matches = None - # self.encoding = 'utf-16' @property def id(self): @@ -55,6 +55,39 @@ class SubtitriIdSubtitle(Subtitle): self.matches = matches return matches + def guess_encoding(self): + # override default subtitle guess_encoding method to not include language-specific encodings guessing + # chardet encoding detection seem to yield better results + """Guess encoding using chardet. + + :return: the guessed encoding. + :rtype: str + + """ + if self._guessed_encoding: + return self._guessed_encoding + + logger.info('Guessing encoding for language %s', self.language) + + # guess/detect encoding using chardet + encoding = chardet.detect(self.content)['encoding'] + logger.info('Chardet found encoding %s', encoding) + + if not encoding: + # fallback on bs4 + logger.info('Falling back to bs4 detection') + a = UnicodeDammit(self.content) + + logger.info("bs4 detected encoding: %s", a.original_encoding) + + if a.original_encoding: + self._guessed_encoding = a.original_encoding + return a.original_encoding + raise ValueError(u"Couldn't guess the proper encoding for %s", self) + + self._guessed_encoding = encoding + return encoding + class SubtitriIdProvider(Provider, ProviderSubtitleArchiveMixin): """subtitri.id.lv Provider.""" @@ -155,7 +188,4 @@ class SubtitriIdProvider(Provider, ProviderSubtitleArchiveMixin): raise ProviderError('Unidentified archive type') - subtitle_content = self.get_subtitle_from_archive(subtitle, archive) - # fix content encoding (utf-16 encoded by default) - fixed_subtitle_content = fix_text(subtitle_content.decode('utf-16'), {'uncurl_quotes': False, 'fix_character_width': False}).encode(encoding='utf-8') - subtitle.content = fixed_subtitle_content + subtitle.content = self.get_subtitle_from_archive(subtitle, archive) From e16b820703e054fd9c7b13098628082cf57f31e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=A2irts=20Kokars?= Date: Wed, 8 May 2019 15:19:46 +0300 Subject: [PATCH 20/23] use download link as the page_link (there is no seperate subtitle page link) --- libs/subliminal_patch/providers/nekur.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/subliminal_patch/providers/nekur.py b/libs/subliminal_patch/providers/nekur.py index f8950d1f2..06fb1401f 100644 --- a/libs/subliminal_patch/providers/nekur.py +++ b/libs/subliminal_patch/providers/nekur.py @@ -160,8 +160,8 @@ class NekurProvider(Provider, ProviderSubtitleArchiveMixin): # additional notes notes = row.select_one('.notes').text.strip() - # page link = archive link (there is no seperate subtitle page link) - page_link = 'http://subtitri.nekur.net/filmu-subtitri/' + # page link = download link (there is no seperate subtitle page link) + page_link = download_link # create/add the subitle subtitle = self.subtitle_class(Language.fromalpha2('lv'), page_link, download_link, title, year, imdb_id, fps, notes) From bddfc5f0af6b1c7f9a39fb7a28919687ece70293 Mon Sep 17 00:00:00 2001 From: Guy Khmelnitsky Date: Wed, 8 May 2019 15:34:51 +0300 Subject: [PATCH 21/23] Update Subscenter to use new Subscenter.biz domain --- libs/subliminal/providers/subscenter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/subliminal/providers/subscenter.py b/libs/subliminal/providers/subscenter.py index f9bf3c8cb..d9c902b7a 100644 --- a/libs/subliminal/providers/subscenter.py +++ b/libs/subliminal/providers/subscenter.py @@ -75,7 +75,7 @@ class SubsCenterSubtitle(Subtitle): class SubsCenterProvider(Provider): """SubsCenter Provider.""" languages = {Language.fromalpha2(l) for l in ['he']} - server_url = 'http://www.subscenter.org/he/' + server_url = 'http://www.subscenter.biz/he/' subtitle_class = SubsCenterSubtitle def __init__(self, username=None, password=None): From 12d942125d5925dc8cd29838b620e45986787aba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Louis=20V=C3=A9zina?= <5130500+morpheus65535@users.noreply.github.com> Date: Fri, 10 May 2019 09:31:39 -0400 Subject: [PATCH 22/23] Version bump. --- bazarr/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bazarr/main.py b/bazarr/main.py index f57fc2e8e..e1218e5e4 100644 --- a/bazarr/main.py +++ b/bazarr/main.py @@ -1,6 +1,6 @@ # coding=utf-8 -bazarr_version = '0.7.4' +bazarr_version = '0.7.5' import gc import sys From 702963901482bc140d43d5d6cf78ff2b78e0033a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Louis=20V=C3=A9zina?= <5130500+morpheus65535@users.noreply.github.com> Date: Fri, 10 May 2019 09:39:47 -0400 Subject: [PATCH 23/23] Fix for anti-captcha class defined even if credentials are empty. --- bazarr/init.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bazarr/init.py b/bazarr/init.py index 2ee74877b..b53755bee 100644 --- a/bazarr/init.py +++ b/bazarr/init.py @@ -17,10 +17,10 @@ from utils import get_binary os.environ["SZ_USER_AGENT"] = "Bazarr/1" # set anti-captcha provider and key -if settings.general.anti_captcha_provider == 'anti-captcha': +if settings.general.anti_captcha_provider == 'anti-captcha' and settings.anticaptcha.anti_captcha_key != "": os.environ["ANTICAPTCHA_CLASS"] = 'AntiCaptchaProxyLess' os.environ["ANTICAPTCHA_ACCOUNT_KEY"] = settings.anticaptcha.anti_captcha_key -elif settings.general.anti_captcha_provider == 'death-by-captcha': +elif settings.general.anti_captcha_provider == 'death-by-captcha' and settings.deathbycaptcha.username != "" and settings.deathbycaptcha.password != "": os.environ["ANTICAPTCHA_CLASS"] = 'DeathByCaptchaProxyLess' os.environ["ANTICAPTCHA_ACCOUNT_KEY"] = ':'.join({settings.deathbycaptcha.username, settings.deathbycaptcha.password}) else: