mirror of https://github.com/tootsuite/mastodon
Ignore low-confidence CharlockHolmes guesses when parsing link cards (#9510)
* Add failing test for windows-1251 link cards * Ignore low-confidence CharlockHolmes guesses Fixes #9466 * Fix no method error when charlock holmes cannot detect charsetpull/9554/head
parent
4ede51743e
commit
e709b8da0d
|
@ -137,7 +137,8 @@ class FetchLinkCardService < BaseService
|
|||
detector.strip_tags = true
|
||||
|
||||
guess = detector.detect(@html, @html_charset)
|
||||
page = Nokogiri::HTML(@html, nil, guess&.fetch(:encoding, nil))
|
||||
encoding = guess&.fetch(:confidence, 0).to_i > 60 ? guess&.fetch(:encoding, nil) : nil
|
||||
page = Nokogiri::HTML(@html, nil, encoding)
|
||||
player_url = meta_property(page, 'twitter:player')
|
||||
|
||||
if player_url && !bad_url?(Addressable::URI.parse(player_url))
|
||||
|
|
|
@ -0,0 +1,17 @@
|
|||
HTTP/1.1 200 OK
|
||||
server: nginx
|
||||
date: Wed, 12 Dec 2018 13:14:03 GMT
|
||||
content-type: text/html
|
||||
content-length: 190
|
||||
accept-ranges: bytes
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=windows-1251" />
|
||||
<title>ñýìïë òåêñò</title>
|
||||
</head>
|
||||
<body>
|
||||
<p>ñýìïë òåêñò</p>
|
||||
</body>
|
||||
</html>
|
|
@ -17,6 +17,8 @@ RSpec.describe FetchLinkCardService, type: :service do
|
|||
stub_request(:head, 'https://github.com/qbi/WannaCry').to_return(status: 404)
|
||||
stub_request(:head, 'http://example.com/test-').to_return(status: 200, headers: { 'Content-Type' => 'text/html' })
|
||||
stub_request(:get, 'http://example.com/test-').to_return(request_fixture('idn.txt'))
|
||||
stub_request(:head, 'http://example.com/windows-1251').to_return(status: 200, headers: { 'Content-Type' => 'text/html' })
|
||||
stub_request(:get, 'http://example.com/windows-1251').to_return(request_fixture('windows-1251.txt'))
|
||||
|
||||
subject.call(status)
|
||||
end
|
||||
|
@ -57,6 +59,15 @@ RSpec.describe FetchLinkCardService, type: :service do
|
|||
end
|
||||
end
|
||||
|
||||
context do
|
||||
let(:status) { Fabricate(:status, text: 'Check out http://example.com/windows-1251') }
|
||||
|
||||
it 'works with windows-1251' do
|
||||
expect(a_request(:get, 'http://example.com/windows-1251')).to have_been_made.at_least_once
|
||||
expect(status.preview_cards.first.title).to eq('сэмпл текст')
|
||||
end
|
||||
end
|
||||
|
||||
context do
|
||||
let(:status) { Fabricate(:status, text: 'テストhttp://example.com/日本語') }
|
||||
|
||||
|
|
Loading…
Reference in New Issue