cjson
.github
workflows CI.yml ci-fuzz.yml
CONTRIBUTING.md
fuzzing
inputs test1 test10 test11 test2 test3 test3.bu test3.uf test3.uu test4 test5 test6 test7 test8 test9
.gitignore CMakeLists.txt afl-prepare-linux.sh afl.c afl.sh cjson_read_fuzzer.c fuzz_main.c json.dict ossfuzz.sh
library_config cJSONConfig.cmake.in cJSONConfigVersion.cmake.in libcjson.pc.in libcjson_utils.pc.in uninstall.cmake
tests
inputs test1 test1.expected test10 test10.expected test11 test11.expected test2 test2.expected test3 test3.expected test4 test4.expected test5 test5.expected test6 test7 test7.expected test8 test8.expected test9 test9.expected
json-patch-tests .editorconfig .gitignore .npmignore README.md cjson-utils-tests.json package.json spec_tests.json tests.json
unity
auto colour_prompt.rb colour_reporter.rb generate_config.yml generate_module.rb generate_test_runner.rb parse_output.rb stylize_as_junit.rb test_file_filter.rb type_sanitizer.rb unity_test_summary.py unity_test_summary.rb unity_to_junit.py
docs ThrowTheSwitchCodingStandard.md UnityAssertionsCheatSheetSuitableforPrintingandPossiblyFraming.pdf UnityAssertionsReference.md UnityConfigurationGuide.md UnityGettingStartedGuide.md UnityHelperScriptsGuide.md license.txt
examples
example_1
src ProductionCode.c ProductionCode.h ProductionCode2.c ProductionCode2.h
makefile readme.txt
example_2
src ProductionCode.c ProductionCode.h ProductionCode2.c ProductionCode2.h
makefile readme.txt
example_3
helper UnityHelper.c UnityHelper.h
src ProductionCode.c ProductionCode.h ProductionCode2.c ProductionCode2.h
rakefile.rb rakefile_helper.rb readme.txt target_gcc_32.yml
unity_config.h
extras
eclipse error_parsers.txt
fixture
src unity_fixture.c unity_fixture.h unity_fixture_internals.h unity_fixture_malloc_overrides.h
rakefile.rb rakefile_helper.rb readme.txt
release build.info version.info
src unity.c unity.h unity_internals.h
.gitattributes .gitignore .travis.yml README.md
CMakeLists.txt cjson_add.c common.h compare_tests.c json_patch_tests.c minify_tests.c misc_tests.c misc_utils_tests.c old_utils_tests.c parse_array.c parse_examples.c parse_hex4.c parse_number.c parse_object.c parse_string.c parse_value.c parse_with_opts.c print_array.c print_number.c print_object.c print_string.c print_value.c readme_examples.c unity_setup.c
.editorconfig .gitattributes .gitignore .travis.yml CHANGELOG.md CMakeLists.txt CONTRIBUTORS.md LICENSE Makefile README.md SECURITY.md appveyor.yml cJSON.c cJSON.h cJSON_Utils.c cJSON_Utils.h test.c valgrind.supp
curl
.circleci config.yml
.github
ISSUE_TEMPLATE bug_report.yml config.yml docs.yml
scripts cleancmd.pl cmp-config.pl cmp-pkg-config.sh codespell-ignore.words codespell.sh distfiles.sh pyspelling.words pyspelling.yaml randcurl.pl requirements-docs.txt requirements-proselint.txt requirements.txt shellcheck-ci.sh shellcheck.sh spellcheck.curl trimmarkdownheader.pl typos.sh typos.toml verify-examples.pl verify-synopsis.pl yamlcheck.sh yamlcheck.yaml
workflows appveyor-status.yml checkdocs.yml checksrc.yml checkurls.yml codeql.yml configure-vs-cmake.yml curl-for-win.yml distcheck.yml fuzz.yml http3-linux.yml label.yml linux-old.yml linux.yml macos.yml non-native.yml windows.yml
CODEOWNERS CONTRIBUTING.md FUNDING.yml dependabot.yml labeler.yml lock.yml stale.yml
CMake CurlSymbolHiding.cmake CurlTests.c FindBrotli.cmake FindCares.cmake FindGSS.cmake FindGnuTLS.cmake FindLDAP.cmake FindLibbacktrace.cmake FindLibgsasl.cmake FindLibidn2.cmake FindLibpsl.cmake FindLibssh.cmake FindLibssh2.cmake FindLibuv.cmake FindMbedTLS.cmake FindNGHTTP2.cmake FindNGHTTP3.cmake FindNGTCP2.cmake FindNettle.cmake FindQuiche.cmake FindRustls.cmake FindWolfSSL.cmake FindZstd.cmake Macros.cmake OtherTests.cmake PickyWarnings.cmake Utilities.cmake cmake_uninstall.in.cmake curl-config.in.cmake unix-cache.cmake win32-cache.cmake
LICENSES BSD-4-Clause-UC.txt ISC.txt curl.txt
docs
cmdline-opts .gitignore CMakeLists.txt MANPAGE.md Makefile.am Makefile.inc _AUTHORS.md _BUGS.md _DESCRIPTION.md _ENVIRONMENT.md _EXITCODES.md _FILES.md _GLOBBING.md _NAME.md _OPTIONS.md _OUTPUT.md _PROGRESS.md _PROTOCOLS.md _PROXYPREFIX.md _SEEALSO.md _SYNOPSIS.md _URL.md _VARIABLES.md _VERSION.md _WWW.md abstract-unix-socket.md alt-svc.md anyauth.md append.md aws-sigv4.md basic.md ca-native.md cacert.md capath.md cert-status.md cert-type.md cert.md ciphers.md compressed-ssh.md compressed.md config.md connect-timeout.md connect-to.md continue-at.md cookie-jar.md cookie.md create-dirs.md create-file-mode.md crlf.md crlfile.md curves.md data-ascii.md data-binary.md data-raw.md data-urlencode.md data.md delegation.md digest.md disable-eprt.md disable-epsv.md disable.md disallow-username-in-url.md dns-interface.md dns-ipv4-addr.md dns-ipv6-addr.md dns-servers.md doh-cert-status.md doh-insecure.md doh-url.md dump-ca-embed.md dump-header.md ech.md egd-file.md engine.md etag-compare.md etag-save.md expect100-timeout.md fail-early.md fail-with-body.md fail.md false-start.md follow.md form-escape.md form-string.md form.md ftp-account.md ftp-alternative-to-user.md ftp-create-dirs.md ftp-method.md ftp-pasv.md ftp-port.md ftp-pret.md ftp-skip-pasv-ip.md ftp-ssl-ccc-mode.md ftp-ssl-ccc.md ftp-ssl-control.md get.md globoff.md happy-eyeballs-timeout-ms.md haproxy-clientip.md haproxy-protocol.md head.md header.md help.md hostpubmd5.md hostpubsha256.md hsts.md http0.9.md http1.0.md http1.1.md http2-prior-knowledge.md http2.md http3-only.md http3.md ignore-content-length.md insecure.md interface.md ip-tos.md ipfs-gateway.md ipv4.md ipv6.md json.md junk-session-cookies.md keepalive-cnt.md keepalive-time.md key-type.md key.md knownhosts.md krb.md libcurl.md limit-rate.md list-only.md local-port.md location-trusted.md location.md login-options.md mail-auth.md mail-from.md mail-rcpt-allowfails.md mail-rcpt.md mainpage.idx manual.md max-filesize.md max-redirs.md max-time.md metalink.md mptcp.md negotiate.md netrc-file.md netrc-optional.md netrc.md next.md no-alpn.md no-buffer.md no-clobber.md no-keepalive.md no-npn.md no-progress-meter.md no-sessionid.md noproxy.md ntlm-wb.md ntlm.md oauth2-bearer.md out-null.md output-dir.md output.md parallel-immediate.md parallel-max-host.md parallel-max.md parallel.md pass.md path-as-is.md pinnedpubkey.md post301.md post302.md post303.md preproxy.md progress-bar.md proto-default.md proto-redir.md proto.md proxy-anyauth.md proxy-basic.md proxy-ca-native.md proxy-cacert.md proxy-capath.md proxy-cert-type.md proxy-cert.md proxy-ciphers.md proxy-crlfile.md proxy-digest.md proxy-header.md proxy-http2.md proxy-insecure.md proxy-key-type.md proxy-key.md proxy-negotiate.md proxy-ntlm.md proxy-pass.md proxy-pinnedpubkey.md proxy-service-name.md proxy-ssl-allow-beast.md proxy-ssl-auto-client-cert.md proxy-tls13-ciphers.md proxy-tlsauthtype.md proxy-tlspassword.md proxy-tlsuser.md proxy-tlsv1.md proxy-user.md proxy.md proxy1.0.md proxytunnel.md pubkey.md quote.md random-file.md range.md rate.md raw.md referer.md remote-header-name.md remote-name-all.md remote-name.md remote-time.md remove-on-error.md request-target.md request.md resolve.md retry-all-errors.md retry-connrefused.md retry-delay.md retry-max-time.md retry.md sasl-authzid.md sasl-ir.md service-name.md show-error.md show-headers.md sigalgs.md silent.md skip-existing.md socks4.md socks4a.md socks5-basic.md socks5-gssapi-nec.md socks5-gssapi-service.md socks5-gssapi.md socks5-hostname.md socks5.md speed-limit.md speed-time.md ssl-allow-beast.md ssl-auto-client-cert.md ssl-no-revoke.md ssl-reqd.md ssl-revoke-best-effort.md ssl-sessions.md ssl.md sslv2.md sslv3.md stderr.md styled-output.md suppress-connect-headers.md tcp-fastopen.md tcp-nodelay.md telnet-option.md tftp-blksize.md tftp-no-options.md time-cond.md tls-earlydata.md tls-max.md tls13-ciphers.md tlsauthtype.md tlspassword.md tlsuser.md tlsv1.0.md tlsv1.1.md tlsv1.2.md tlsv1.3.md tlsv1.md tr-encoding.md trace-ascii.md trace-config.md trace-ids.md trace-time.md trace.md unix-socket.md upload-file.md upload-flags.md url-query.md url.md use-ascii.md user-agent.md user.md variable.md verbose.md version.md vlan-priority.md write-out.md xattr.md
examples .checksrc .gitignore 10-at-a-time.c CMakeLists.txt Makefile.am Makefile.example Makefile.inc README.md adddocsref.pl address-scope.c altsvc.c anyauthput.c block_ip.c cacertinmem.c certinfo.c chkspeed.c connect-to.c cookie_interface.c crawler.c debug.c default-scheme.c ephiperfifo.c evhiperfifo.c externalsocket.c fileupload.c ftp-delete.c ftp-wildcard.c ftpget.c ftpgetinfo.c ftpgetresp.c ftpsget.c ftpupload.c ftpuploadfrommem.c ftpuploadresume.c getinfo.c getinmemory.c getredirect.c getreferrer.c ghiper.c headerapi.c hiperfifo.c hsts-preload.c htmltidy.c htmltitle.cpp http-options.c http-post.c http2-download.c http2-pushinmemory.c http2-serverpush.c http2-upload.c http3-present.c http3.c httpcustomheader.c httpput-postfields.c httpput.c https.c imap-append.c imap-authzid.c imap-copy.c imap-create.c imap-delete.c imap-examine.c imap-fetch.c imap-list.c imap-lsub.c imap-multi.c imap-noop.c imap-search.c imap-ssl.c imap-store.c imap-tls.c interface.c ipv6.c keepalive.c localport.c log_failed_transfers.c maxconnects.c multi-app.c multi-debugcallback.c multi-double.c multi-event.c multi-formadd.c multi-legacy.c multi-post.c multi-single.c multi-uv.c netrc.c parseurl.c persistent.c pop3-authzid.c pop3-dele.c pop3-list.c pop3-multi.c pop3-noop.c pop3-retr.c pop3-ssl.c pop3-stat.c pop3-tls.c pop3-top.c pop3-uidl.c post-callback.c postinmemory.c postit2-formadd.c postit2.c progressfunc.c protofeats.c range.c resolve.c rtsp-options.c sendrecv.c sepheaders.c sessioninfo.c sftpget.c sftpuploadresume.c shared-connection-cache.c simple.c simplepost.c simplessl.c smooth-gtk-thread.c smtp-authzid.c smtp-expn.c smtp-mail.c smtp-mime.c smtp-multi.c smtp-ssl.c smtp-tls.c smtp-vrfy.c sslbackend.c synctime.c threaded.c unixsocket.c url2file.c urlapi.c usercertinmem.c version-check.pl websocket-cb.c websocket-updown.c websocket.c xmlstream.c
internals BUFQ.md BUFREF.md CHECKSRC.md CLIENT-READERS.md CLIENT-WRITERS.md CODE_STYLE.md CONNECTION-FILTERS.md CREDENTIALS.md CURLX.md DYNBUF.md HASH.md LLIST.md MID.md MQTT.md MULTI-EV.md NEW-PROTOCOL.md PEERS.md PORTING.md RATELIMITS.md README.md SCORECARD.md SPLAY.md STRPARSE.md THRDPOOL-AND-QUEUE.md TIME-KEEPING.md TLS-SESSIONS.md UINT_SETS.md WEBSOCKET.md
libcurl
opts CMakeLists.txt CURLINFO_ACTIVESOCKET.md CURLINFO_APPCONNECT_TIME.md CURLINFO_APPCONNECT_TIME_T.md CURLINFO_CAINFO.md CURLINFO_CAPATH.md CURLINFO_CERTINFO.md CURLINFO_CONDITION_UNMET.md CURLINFO_CONNECT_TIME.md CURLINFO_CONNECT_TIME_T.md CURLINFO_CONN_ID.md CURLINFO_CONTENT_LENGTH_DOWNLOAD.md CURLINFO_CONTENT_LENGTH_DOWNLOAD_T.md CURLINFO_CONTENT_LENGTH_UPLOAD.md CURLINFO_CONTENT_LENGTH_UPLOAD_T.md CURLINFO_CONTENT_TYPE.md CURLINFO_COOKIELIST.md CURLINFO_EARLYDATA_SENT_T.md CURLINFO_EFFECTIVE_METHOD.md CURLINFO_EFFECTIVE_URL.md CURLINFO_FILETIME.md CURLINFO_FILETIME_T.md CURLINFO_FTP_ENTRY_PATH.md CURLINFO_HEADER_SIZE.md CURLINFO_HTTPAUTH_AVAIL.md CURLINFO_HTTPAUTH_USED.md CURLINFO_HTTP_CONNECTCODE.md CURLINFO_HTTP_VERSION.md CURLINFO_LASTSOCKET.md CURLINFO_LOCAL_IP.md CURLINFO_LOCAL_PORT.md CURLINFO_NAMELOOKUP_TIME.md CURLINFO_NAMELOOKUP_TIME_T.md CURLINFO_NUM_CONNECTS.md CURLINFO_OS_ERRNO.md CURLINFO_POSTTRANSFER_TIME_T.md CURLINFO_PRETRANSFER_TIME.md CURLINFO_PRETRANSFER_TIME_T.md CURLINFO_PRIMARY_IP.md CURLINFO_PRIMARY_PORT.md CURLINFO_PRIVATE.md CURLINFO_PROTOCOL.md CURLINFO_PROXYAUTH_AVAIL.md CURLINFO_PROXYAUTH_USED.md CURLINFO_PROXY_ERROR.md CURLINFO_PROXY_SSL_VERIFYRESULT.md CURLINFO_QUEUE_TIME_T.md CURLINFO_REDIRECT_COUNT.md CURLINFO_REDIRECT_TIME.md CURLINFO_REDIRECT_TIME_T.md CURLINFO_REDIRECT_URL.md CURLINFO_REFERER.md CURLINFO_REQUEST_SIZE.md CURLINFO_RESPONSE_CODE.md CURLINFO_RETRY_AFTER.md CURLINFO_RTSP_CLIENT_CSEQ.md CURLINFO_RTSP_CSEQ_RECV.md CURLINFO_RTSP_SERVER_CSEQ.md CURLINFO_RTSP_SESSION_ID.md CURLINFO_SCHEME.md CURLINFO_SIZE_DELIVERED.md CURLINFO_SIZE_DOWNLOAD.md CURLINFO_SIZE_DOWNLOAD_T.md CURLINFO_SIZE_UPLOAD.md CURLINFO_SIZE_UPLOAD_T.md CURLINFO_SPEED_DOWNLOAD.md CURLINFO_SPEED_DOWNLOAD_T.md CURLINFO_SPEED_UPLOAD.md CURLINFO_SPEED_UPLOAD_T.md CURLINFO_SSL_ENGINES.md CURLINFO_SSL_VERIFYRESULT.md CURLINFO_STARTTRANSFER_TIME.md CURLINFO_STARTTRANSFER_TIME_T.md CURLINFO_TLS_SESSION.md CURLINFO_TLS_SSL_PTR.md CURLINFO_TOTAL_TIME.md CURLINFO_TOTAL_TIME_T.md CURLINFO_USED_PROXY.md CURLINFO_XFER_ID.md CURLMINFO_XFERS_ADDED.md CURLMINFO_XFERS_CURRENT.md CURLMINFO_XFERS_DONE.md CURLMINFO_XFERS_PENDING.md CURLMINFO_XFERS_RUNNING.md CURLMOPT_CHUNK_LENGTH_PENALTY_SIZE.md CURLMOPT_CONTENT_LENGTH_PENALTY_SIZE.md CURLMOPT_MAXCONNECTS.md CURLMOPT_MAX_CONCURRENT_STREAMS.md CURLMOPT_MAX_HOST_CONNECTIONS.md CURLMOPT_MAX_PIPELINE_LENGTH.md CURLMOPT_MAX_TOTAL_CONNECTIONS.md CURLMOPT_NETWORK_CHANGED.md CURLMOPT_NOTIFYDATA.md CURLMOPT_NOTIFYFUNCTION.md CURLMOPT_PIPELINING.md CURLMOPT_PIPELINING_SERVER_BL.md CURLMOPT_PIPELINING_SITE_BL.md CURLMOPT_PUSHDATA.md CURLMOPT_PUSHFUNCTION.md CURLMOPT_QUICK_EXIT.md CURLMOPT_RESOLVE_THREADS_MAX.md CURLMOPT_SOCKETDATA.md CURLMOPT_SOCKETFUNCTION.md CURLMOPT_TIMERDATA.md CURLMOPT_TIMERFUNCTION.md CURLOPT_ABSTRACT_UNIX_SOCKET.md CURLOPT_ACCEPTTIMEOUT_MS.md CURLOPT_ACCEPT_ENCODING.md CURLOPT_ADDRESS_SCOPE.md CURLOPT_ALTSVC.md CURLOPT_ALTSVC_CTRL.md CURLOPT_APPEND.md CURLOPT_AUTOREFERER.md CURLOPT_AWS_SIGV4.md CURLOPT_BUFFERSIZE.md CURLOPT_CAINFO.md CURLOPT_CAINFO_BLOB.md CURLOPT_CAPATH.md CURLOPT_CA_CACHE_TIMEOUT.md CURLOPT_CERTINFO.md CURLOPT_CHUNK_BGN_FUNCTION.md CURLOPT_CHUNK_DATA.md CURLOPT_CHUNK_END_FUNCTION.md CURLOPT_CLOSESOCKETDATA.md CURLOPT_CLOSESOCKETFUNCTION.md CURLOPT_CONNECTTIMEOUT.md CURLOPT_CONNECTTIMEOUT_MS.md CURLOPT_CONNECT_ONLY.md CURLOPT_CONNECT_TO.md CURLOPT_CONV_FROM_NETWORK_FUNCTION.md CURLOPT_CONV_FROM_UTF8_FUNCTION.md CURLOPT_CONV_TO_NETWORK_FUNCTION.md CURLOPT_COOKIE.md CURLOPT_COOKIEFILE.md CURLOPT_COOKIEJAR.md CURLOPT_COOKIELIST.md CURLOPT_COOKIESESSION.md CURLOPT_COPYPOSTFIELDS.md CURLOPT_CRLF.md CURLOPT_CRLFILE.md CURLOPT_CURLU.md CURLOPT_CUSTOMREQUEST.md CURLOPT_DEBUGDATA.md CURLOPT_DEBUGFUNCTION.md CURLOPT_DEFAULT_PROTOCOL.md CURLOPT_DIRLISTONLY.md CURLOPT_DISALLOW_USERNAME_IN_URL.md CURLOPT_DNS_CACHE_TIMEOUT.md CURLOPT_DNS_INTERFACE.md CURLOPT_DNS_LOCAL_IP4.md CURLOPT_DNS_LOCAL_IP6.md CURLOPT_DNS_SERVERS.md CURLOPT_DNS_SHUFFLE_ADDRESSES.md CURLOPT_DNS_USE_GLOBAL_CACHE.md CURLOPT_DOH_SSL_VERIFYHOST.md CURLOPT_DOH_SSL_VERIFYPEER.md CURLOPT_DOH_SSL_VERIFYSTATUS.md CURLOPT_DOH_URL.md CURLOPT_ECH.md CURLOPT_EGDSOCKET.md CURLOPT_ERRORBUFFER.md CURLOPT_EXPECT_100_TIMEOUT_MS.md CURLOPT_FAILONERROR.md CURLOPT_FILETIME.md CURLOPT_FNMATCH_DATA.md CURLOPT_FNMATCH_FUNCTION.md CURLOPT_FOLLOWLOCATION.md CURLOPT_FORBID_REUSE.md CURLOPT_FRESH_CONNECT.md CURLOPT_FTPPORT.md CURLOPT_FTPSSLAUTH.md CURLOPT_FTP_ACCOUNT.md CURLOPT_FTP_ALTERNATIVE_TO_USER.md CURLOPT_FTP_CREATE_MISSING_DIRS.md CURLOPT_FTP_FILEMETHOD.md CURLOPT_FTP_SKIP_PASV_IP.md CURLOPT_FTP_SSL_CCC.md CURLOPT_FTP_USE_EPRT.md CURLOPT_FTP_USE_EPSV.md CURLOPT_FTP_USE_PRET.md CURLOPT_GSSAPI_DELEGATION.md CURLOPT_HAPPY_EYEBALLS_TIMEOUT_MS.md CURLOPT_HAPROXYPROTOCOL.md CURLOPT_HAPROXY_CLIENT_IP.md CURLOPT_HEADER.md CURLOPT_HEADERDATA.md CURLOPT_HEADERFUNCTION.md CURLOPT_HEADEROPT.md CURLOPT_HSTS.md CURLOPT_HSTSREADDATA.md CURLOPT_HSTSREADFUNCTION.md CURLOPT_HSTSWRITEDATA.md CURLOPT_HSTSWRITEFUNCTION.md CURLOPT_HSTS_CTRL.md CURLOPT_HTTP09_ALLOWED.md CURLOPT_HTTP200ALIASES.md CURLOPT_HTTPAUTH.md CURLOPT_HTTPGET.md CURLOPT_HTTPHEADER.md CURLOPT_HTTPPOST.md CURLOPT_HTTPPROXYTUNNEL.md CURLOPT_HTTP_CONTENT_DECODING.md CURLOPT_HTTP_TRANSFER_DECODING.md CURLOPT_HTTP_VERSION.md CURLOPT_IGNORE_CONTENT_LENGTH.md CURLOPT_INFILESIZE.md CURLOPT_INFILESIZE_LARGE.md CURLOPT_INTERFACE.md CURLOPT_INTERLEAVEDATA.md CURLOPT_INTERLEAVEFUNCTION.md CURLOPT_IOCTLDATA.md CURLOPT_IOCTLFUNCTION.md CURLOPT_IPRESOLVE.md CURLOPT_ISSUERCERT.md CURLOPT_ISSUERCERT_BLOB.md CURLOPT_KEEP_SENDING_ON_ERROR.md CURLOPT_KEYPASSWD.md CURLOPT_KRBLEVEL.md CURLOPT_LOCALPORT.md CURLOPT_LOCALPORTRANGE.md CURLOPT_LOGIN_OPTIONS.md CURLOPT_LOW_SPEED_LIMIT.md CURLOPT_LOW_SPEED_TIME.md CURLOPT_MAIL_AUTH.md CURLOPT_MAIL_FROM.md CURLOPT_MAIL_RCPT.md CURLOPT_MAIL_RCPT_ALLOWFAILS.md CURLOPT_MAXAGE_CONN.md CURLOPT_MAXCONNECTS.md CURLOPT_MAXFILESIZE.md CURLOPT_MAXFILESIZE_LARGE.md CURLOPT_MAXLIFETIME_CONN.md CURLOPT_MAXREDIRS.md CURLOPT_MAX_RECV_SPEED_LARGE.md CURLOPT_MAX_SEND_SPEED_LARGE.md CURLOPT_MIMEPOST.md CURLOPT_MIME_OPTIONS.md CURLOPT_NETRC.md CURLOPT_NETRC_FILE.md CURLOPT_NEW_DIRECTORY_PERMS.md CURLOPT_NEW_FILE_PERMS.md CURLOPT_NOBODY.md CURLOPT_NOPROGRESS.md CURLOPT_NOPROXY.md CURLOPT_NOSIGNAL.md CURLOPT_OPENSOCKETDATA.md CURLOPT_OPENSOCKETFUNCTION.md CURLOPT_PASSWORD.md CURLOPT_PATH_AS_IS.md CURLOPT_PINNEDPUBLICKEY.md CURLOPT_PIPEWAIT.md CURLOPT_PORT.md CURLOPT_POST.md CURLOPT_POSTFIELDS.md CURLOPT_POSTFIELDSIZE.md CURLOPT_POSTFIELDSIZE_LARGE.md CURLOPT_POSTQUOTE.md CURLOPT_POSTREDIR.md CURLOPT_PREQUOTE.md CURLOPT_PREREQDATA.md CURLOPT_PREREQFUNCTION.md CURLOPT_PRE_PROXY.md CURLOPT_PRIVATE.md CURLOPT_PROGRESSDATA.md CURLOPT_PROGRESSFUNCTION.md CURLOPT_PROTOCOLS.md CURLOPT_PROTOCOLS_STR.md CURLOPT_PROXY.md CURLOPT_PROXYAUTH.md CURLOPT_PROXYHEADER.md CURLOPT_PROXYPASSWORD.md CURLOPT_PROXYPORT.md CURLOPT_PROXYTYPE.md CURLOPT_PROXYUSERNAME.md CURLOPT_PROXYUSERPWD.md CURLOPT_PROXY_CAINFO.md CURLOPT_PROXY_CAINFO_BLOB.md CURLOPT_PROXY_CAPATH.md CURLOPT_PROXY_CRLFILE.md CURLOPT_PROXY_ISSUERCERT.md CURLOPT_PROXY_ISSUERCERT_BLOB.md CURLOPT_PROXY_KEYPASSWD.md CURLOPT_PROXY_PINNEDPUBLICKEY.md CURLOPT_PROXY_SERVICE_NAME.md CURLOPT_PROXY_SSLCERT.md CURLOPT_PROXY_SSLCERTTYPE.md CURLOPT_PROXY_SSLCERT_BLOB.md CURLOPT_PROXY_SSLKEY.md CURLOPT_PROXY_SSLKEYTYPE.md CURLOPT_PROXY_SSLKEY_BLOB.md CURLOPT_PROXY_SSLVERSION.md CURLOPT_PROXY_SSL_CIPHER_LIST.md CURLOPT_PROXY_SSL_OPTIONS.md CURLOPT_PROXY_SSL_VERIFYHOST.md CURLOPT_PROXY_SSL_VERIFYPEER.md CURLOPT_PROXY_TLS13_CIPHERS.md CURLOPT_PROXY_TLSAUTH_PASSWORD.md CURLOPT_PROXY_TLSAUTH_TYPE.md CURLOPT_PROXY_TLSAUTH_USERNAME.md CURLOPT_PROXY_TRANSFER_MODE.md CURLOPT_PUT.md CURLOPT_QUICK_EXIT.md CURLOPT_QUOTE.md CURLOPT_RANDOM_FILE.md CURLOPT_RANGE.md CURLOPT_READDATA.md CURLOPT_READFUNCTION.md CURLOPT_REDIR_PROTOCOLS.md CURLOPT_REDIR_PROTOCOLS_STR.md CURLOPT_REFERER.md CURLOPT_REQUEST_TARGET.md CURLOPT_RESOLVE.md CURLOPT_RESOLVER_START_DATA.md CURLOPT_RESOLVER_START_FUNCTION.md CURLOPT_RESUME_FROM.md CURLOPT_RESUME_FROM_LARGE.md CURLOPT_RTSP_CLIENT_CSEQ.md CURLOPT_RTSP_REQUEST.md CURLOPT_RTSP_SERVER_CSEQ.md CURLOPT_RTSP_SESSION_ID.md CURLOPT_RTSP_STREAM_URI.md CURLOPT_RTSP_TRANSPORT.md CURLOPT_SASL_AUTHZID.md CURLOPT_SASL_IR.md CURLOPT_SEEKDATA.md CURLOPT_SEEKFUNCTION.md CURLOPT_SERVER_RESPONSE_TIMEOUT.md CURLOPT_SERVER_RESPONSE_TIMEOUT_MS.md CURLOPT_SERVICE_NAME.md CURLOPT_SHARE.md CURLOPT_SOCKOPTDATA.md CURLOPT_SOCKOPTFUNCTION.md CURLOPT_SOCKS5_AUTH.md CURLOPT_SOCKS5_GSSAPI_NEC.md CURLOPT_SOCKS5_GSSAPI_SERVICE.md CURLOPT_SSH_AUTH_TYPES.md CURLOPT_SSH_COMPRESSION.md CURLOPT_SSH_HOSTKEYDATA.md CURLOPT_SSH_HOSTKEYFUNCTION.md CURLOPT_SSH_HOST_PUBLIC_KEY_MD5.md CURLOPT_SSH_HOST_PUBLIC_KEY_SHA256.md CURLOPT_SSH_KEYDATA.md CURLOPT_SSH_KEYFUNCTION.md CURLOPT_SSH_KNOWNHOSTS.md CURLOPT_SSH_PRIVATE_KEYFILE.md CURLOPT_SSH_PUBLIC_KEYFILE.md CURLOPT_SSLCERT.md CURLOPT_SSLCERTTYPE.md CURLOPT_SSLCERT_BLOB.md CURLOPT_SSLENGINE.md CURLOPT_SSLENGINE_DEFAULT.md CURLOPT_SSLKEY.md CURLOPT_SSLKEYTYPE.md CURLOPT_SSLKEY_BLOB.md CURLOPT_SSLVERSION.md CURLOPT_SSL_CIPHER_LIST.md CURLOPT_SSL_CTX_DATA.md CURLOPT_SSL_CTX_FUNCTION.md CURLOPT_SSL_EC_CURVES.md CURLOPT_SSL_ENABLE_ALPN.md CURLOPT_SSL_ENABLE_NPN.md CURLOPT_SSL_FALSESTART.md CURLOPT_SSL_OPTIONS.md CURLOPT_SSL_SESSIONID_CACHE.md CURLOPT_SSL_SIGNATURE_ALGORITHMS.md CURLOPT_SSL_VERIFYHOST.md CURLOPT_SSL_VERIFYPEER.md CURLOPT_SSL_VERIFYSTATUS.md CURLOPT_STDERR.md CURLOPT_STREAM_DEPENDS.md CURLOPT_STREAM_DEPENDS_E.md CURLOPT_STREAM_WEIGHT.md CURLOPT_SUPPRESS_CONNECT_HEADERS.md CURLOPT_TCP_FASTOPEN.md CURLOPT_TCP_KEEPALIVE.md CURLOPT_TCP_KEEPCNT.md CURLOPT_TCP_KEEPIDLE.md CURLOPT_TCP_KEEPINTVL.md CURLOPT_TCP_NODELAY.md CURLOPT_TELNETOPTIONS.md CURLOPT_TFTP_BLKSIZE.md CURLOPT_TFTP_NO_OPTIONS.md CURLOPT_TIMECONDITION.md CURLOPT_TIMEOUT.md CURLOPT_TIMEOUT_MS.md CURLOPT_TIMEVALUE.md CURLOPT_TIMEVALUE_LARGE.md CURLOPT_TLS13_CIPHERS.md CURLOPT_TLSAUTH_PASSWORD.md CURLOPT_TLSAUTH_TYPE.md CURLOPT_TLSAUTH_USERNAME.md CURLOPT_TRAILERDATA.md CURLOPT_TRAILERFUNCTION.md CURLOPT_TRANSFERTEXT.md CURLOPT_TRANSFER_ENCODING.md CURLOPT_UNIX_SOCKET_PATH.md CURLOPT_UNRESTRICTED_AUTH.md CURLOPT_UPKEEP_INTERVAL_MS.md CURLOPT_UPLOAD.md CURLOPT_UPLOAD_BUFFERSIZE.md CURLOPT_UPLOAD_FLAGS.md CURLOPT_URL.md CURLOPT_USERAGENT.md CURLOPT_USERNAME.md CURLOPT_USERPWD.md CURLOPT_USE_SSL.md CURLOPT_VERBOSE.md CURLOPT_WILDCARDMATCH.md CURLOPT_WRITEDATA.md CURLOPT_WRITEFUNCTION.md CURLOPT_WS_OPTIONS.md CURLOPT_XFERINFODATA.md CURLOPT_XFERINFOFUNCTION.md CURLOPT_XOAUTH2_BEARER.md CURLSHOPT_LOCKFUNC.md CURLSHOPT_SHARE.md CURLSHOPT_UNLOCKFUNC.md CURLSHOPT_UNSHARE.md CURLSHOPT_USERDATA.md Makefile.am Makefile.inc
.gitignore ABI.md CMakeLists.txt Makefile.am Makefile.inc curl_easy_cleanup.md curl_easy_duphandle.md curl_easy_escape.md curl_easy_getinfo.md curl_easy_header.md curl_easy_init.md curl_easy_nextheader.md curl_easy_option_by_id.md curl_easy_option_by_name.md curl_easy_option_next.md curl_easy_pause.md curl_easy_perform.md curl_easy_recv.md curl_easy_reset.md curl_easy_send.md curl_easy_setopt.md curl_easy_ssls_export.md curl_easy_ssls_import.md curl_easy_strerror.md curl_easy_unescape.md curl_easy_upkeep.md curl_escape.md curl_formadd.md curl_formfree.md curl_formget.md curl_free.md curl_getdate.md curl_getenv.md curl_global_cleanup.md curl_global_init.md curl_global_init_mem.md curl_global_sslset.md curl_global_trace.md curl_mime_addpart.md curl_mime_data.md curl_mime_data_cb.md curl_mime_encoder.md curl_mime_filedata.md curl_mime_filename.md curl_mime_free.md curl_mime_headers.md curl_mime_init.md curl_mime_name.md curl_mime_subparts.md curl_mime_type.md curl_mprintf.md curl_multi_add_handle.md curl_multi_assign.md curl_multi_cleanup.md curl_multi_fdset.md curl_multi_get_handles.md curl_multi_get_offt.md curl_multi_info_read.md curl_multi_init.md curl_multi_notify_disable.md curl_multi_notify_enable.md curl_multi_perform.md curl_multi_poll.md curl_multi_remove_handle.md curl_multi_setopt.md curl_multi_socket.md curl_multi_socket_action.md curl_multi_socket_all.md curl_multi_strerror.md curl_multi_timeout.md curl_multi_wait.md curl_multi_waitfds.md curl_multi_wakeup.md curl_pushheader_byname.md curl_pushheader_bynum.md curl_share_cleanup.md curl_share_init.md curl_share_setopt.md curl_share_strerror.md curl_slist_append.md curl_slist_free_all.md curl_strequal.md curl_strnequal.md curl_unescape.md curl_url.md curl_url_cleanup.md curl_url_dup.md curl_url_get.md curl_url_set.md curl_url_strerror.md curl_version.md curl_version_info.md curl_ws_meta.md curl_ws_recv.md curl_ws_send.md curl_ws_start_frame.md libcurl-easy.md libcurl-env-dbg.md libcurl-env.md libcurl-errors.md libcurl-multi.md libcurl-security.md libcurl-share.md libcurl-thread.md libcurl-tutorial.md libcurl-url.md libcurl-ws.md libcurl.m4 libcurl.md mksymbolsmanpage.pl symbols-in-versions symbols.pl
tests CI.md FILEFORMAT.md HTTP.md TEST-SUITE.md
.gitignore ALTSVC.md BINDINGS.md BUG-BOUNTY.md BUGS.md CIPHERS-TLS12.md CIPHERS.md CMakeLists.txt CODE_OF_CONDUCT.md CODE_REVIEW.md CONTRIBUTE.md CURL-DISABLE.md CURLDOWN.md DEPRECATE.md DISTROS.md EARLY-RELEASE.md ECH.md EXPERIMENTAL.md FAQ.md FEATURES.md GOVERNANCE.md HELP-US.md HISTORY.md HSTS.md HTTP-COOKIES.md HTTP3.md HTTPSRR.md INFRASTRUCTURE.md INSTALL-CMAKE.md INSTALL.md INTERNALS.md IPFS.md KNOWN_BUGS.md KNOWN_RISKS.md MAIL-ETIQUETTE.md MANUAL.md Makefile.am README.md RELEASE-PROCEDURE.md ROADMAP.md RUSTLS.md SECURITY-ADVISORY.md SPONSORS.md SSL-PROBLEMS.md SSLCERTS.md THANKS THANKS-filter TODO.md TheArtOfHttpScripting.md URL-SYNTAX.md VERIFY.md VERSIONS.md VULN-DISCLOSURE-POLICY.md curl-config.md mk-ca-bundle.md options-in-versions runtests.md testcurl.md wcurl.md
include
curl Makefile.am curl.h curlver.h easy.h header.h mprintf.h multi.h options.h stdcheaders.h system.h typecheck-gcc.h urlapi.h websockets.h
Makefile.am README.md
lib
curlx base64.c base64.h basename.c basename.h dynbuf.c dynbuf.h fopen.c fopen.h inet_ntop.c inet_ntop.h inet_pton.c inet_pton.h multibyte.c multibyte.h nonblock.c nonblock.h snprintf.c snprintf.h strcopy.c strcopy.h strdup.c strdup.h strerr.c strerr.h strparse.c strparse.h timediff.c timediff.h timeval.c timeval.h version_win32.c version_win32.h wait.c wait.h warnless.c warnless.h winapi.c winapi.h
vauth cleartext.c cram.c digest.c digest.h digest_sspi.c gsasl.c krb5_gssapi.c krb5_sspi.c ntlm.c ntlm_sspi.c oauth2.c spnego_gssapi.c spnego_sspi.c vauth.c vauth.h
vquic curl_ngtcp2.c curl_ngtcp2.h curl_quiche.c curl_quiche.h vquic-tls.c vquic-tls.h vquic.c vquic.h vquic_int.h
vssh libssh.c libssh2.c ssh.h vssh.c vssh.h
vtls apple.c apple.h cipher_suite.c cipher_suite.h gtls.c gtls.h hostcheck.c hostcheck.h keylog.c keylog.h mbedtls.c mbedtls.h openssl.c openssl.h rustls.c rustls.h schannel.c schannel.h schannel_int.h schannel_verify.c vtls.c vtls.h vtls_int.h vtls_scache.c vtls_scache.h vtls_spack.c vtls_spack.h wolfssl.c wolfssl.h x509asn1.c x509asn1.h
.gitignore CMakeLists.txt Makefile.am Makefile.inc Makefile.soname altsvc.c altsvc.h amigaos.c amigaos.h arpa_telnet.h asyn-ares.c asyn-base.c asyn-thrdd.c asyn.h bufq.c bufq.h bufref.c bufref.h cf-dns.c cf-dns.h cf-h1-proxy.c cf-h1-proxy.h cf-h2-proxy.c cf-h2-proxy.h cf-haproxy.c cf-haproxy.h cf-https-connect.c cf-https-connect.h cf-ip-happy.c cf-ip-happy.h cf-socket.c cf-socket.h cfilters.c cfilters.h config-mac.h config-os400.h config-riscos.h config-win32.h conncache.c conncache.h connect.c connect.h content_encoding.c content_encoding.h cookie.c cookie.h creds.c creds.h cshutdn.c cshutdn.h curl_addrinfo.c curl_addrinfo.h curl_config-cmake.h.in curl_ctype.h curl_endian.c curl_endian.h curl_fnmatch.c curl_fnmatch.h curl_fopen.c curl_fopen.h curl_get_line.c curl_get_line.h curl_gethostname.c curl_gethostname.h curl_gssapi.c curl_gssapi.h curl_hmac.h curl_ldap.h curl_md4.h curl_md5.h curl_memrchr.c curl_memrchr.h curl_ntlm_core.c curl_ntlm_core.h curl_printf.h curl_range.c curl_range.h curl_sasl.c curl_sasl.h curl_setup.h curl_sha256.h curl_sha512_256.c curl_sha512_256.h curl_share.c curl_share.h curl_sspi.c curl_sspi.h curl_threads.c curl_threads.h curl_trc.c curl_trc.h cw-out.c cw-out.h cw-pause.c cw-pause.h dict.c dict.h dllmain.c dnscache.c dnscache.h doh.c doh.h dynhds.c dynhds.h easy.c easy_lock.h easygetopt.c easyif.h easyoptions.c easyoptions.h escape.c escape.h fake_addrinfo.c fake_addrinfo.h file.c file.h fileinfo.c fileinfo.h formdata.c formdata.h ftp-int.h ftp.c ftp.h ftplistparser.c ftplistparser.h functypes.h getenv.c getinfo.c getinfo.h gopher.c gopher.h hash.c hash.h headers.c headers.h hmac.c hostip.c hostip.h hostip4.c hostip6.c hsts.c hsts.h http.c http.h http1.c http1.h http2.c http2.h http_aws_sigv4.c http_aws_sigv4.h http_chunks.c http_chunks.h http_digest.c http_digest.h http_negotiate.c http_negotiate.h http_ntlm.c http_ntlm.h http_proxy.c http_proxy.h httpsrr.c httpsrr.h idn.c idn.h if2ip.c if2ip.h imap.c imap.h ldap.c libcurl.def libcurl.rc libcurl.vers.in llist.c llist.h macos.c macos.h md4.c md5.c memdebug.c mime.c mime.h mprintf.c mqtt.c mqtt.h multi.c multi_ev.c multi_ev.h multi_ntfy.c multi_ntfy.h multihandle.h multiif.h netrc.c netrc.h noproxy.c noproxy.h openldap.c optiontable.pl parsedate.c parsedate.h peer.c peer.h pingpong.c pingpong.h pop3.c pop3.h progress.c progress.h protocol.c protocol.h psl.c psl.h rand.c rand.h ratelimit.c ratelimit.h request.c request.h rtsp.c rtsp.h select.c select.h sendf.c sendf.h setopt.c setopt.h setup-os400.h setup-vms.h setup-win32.h sha256.c sigpipe.h slist.c slist.h smb.c smb.h smtp.c smtp.h sockaddr.h socketpair.c socketpair.h socks.c socks.h socks_gssapi.c socks_sspi.c splay.c splay.h strcase.c strcase.h strequal.c strerror.c strerror.h system_win32.c system_win32.h telnet.c telnet.h tftp.c tftp.h thrdpool.c thrdpool.h thrdqueue.c thrdqueue.h transfer.c transfer.h uint-bset.c uint-bset.h uint-hash.c uint-hash.h uint-spbset.c uint-spbset.h uint-table.c uint-table.h url.c url.h urlapi-int.h urlapi.c urldata.h version.c ws.c ws.h
m4 .gitignore curl-amissl.m4 curl-apple-sectrust.m4 curl-compilers.m4 curl-confopts.m4 curl-functions.m4 curl-gnutls.m4 curl-mbedtls.m4 curl-openssl.m4 curl-override.m4 curl-reentrant.m4 curl-rustls.m4 curl-schannel.m4 curl-sysconfig.m4 curl-wolfssl.m4 xc-am-iface.m4 xc-cc-check.m4 xc-lt-iface.m4 xc-val-flgs.m4 zz40-xc-ovr.m4 zz50-xc-ovr.m4
projects
OS400
rpg-examples HEADERAPI HTTPPOST INMEMORY SIMPLE1 SIMPLE2 SMTPSRCMBR
.checksrc README.OS400 ccsidcurl.c ccsidcurl.h config400.default curl.cmd curl.inc.in curlcl.c curlmain.c initscript.sh make-docs.sh make-include.sh make-lib.sh make-src.sh make-tests.sh makefile.sh os400sys.c os400sys.h
Windows
tmpl .gitattributes README.txt curl-all.sln curl.sln curl.vcxproj curl.vcxproj.filters libcurl.sln libcurl.vcxproj libcurl.vcxproj.filters
.gitignore README.md generate.bat
vms Makefile.am backup_gnv_curl_src.com build_curl-config_script.com build_gnv_curl.com build_gnv_curl_pcsi_desc.com build_gnv_curl_pcsi_text.com build_gnv_curl_release_notes.com build_libcurl_pc.com build_vms.com clean_gnv_curl.com compare_curl_source.com config_h.com curl_crtl_init.c curl_gnv_build_steps.txt curl_release_note_start.txt curl_startup.com curlmsg.h curlmsg.msg curlmsg.sdl curlmsg_vms.h generate_config_vms_h_curl.com generate_vax_transfer.com gnv_conftest.c_first gnv_curl_configure.sh gnv_libcurl_symbols.opt gnv_link_curl.com macro32_exactcase.patch make_gnv_curl_install.sh make_pcsi_curl_kit_name.com pcsi_gnv_curl_file_list.txt pcsi_product_gnv_curl.com readme report_openssl_version.c setup_gnv_curl_build.com stage_curl_install.com vms_eco_level.h
Makefile.am README.md
scripts .checksrc CMakeLists.txt Makefile.am badwords badwords-all badwords.txt cd2cd cd2nroff cdall checksrc-all.pl checksrc.pl cmakelint.sh completion.pl contributors.sh contrithanks.sh coverage.sh delta dmaketgz extract-unit-protos firefox-db2pem.sh installcheck.sh maketgz managen mdlinkcheck mk-ca-bundle.pl mk-unity.pl nroff2cd perlcheck.sh pythonlint.sh randdisable release-notes.pl release-tools.sh schemetable.c singleuse.pl spacecheck.pl top-complexity top-length verify-release wcurl
src
toolx tool_time.c tool_time.h
.checksrc .gitignore CMakeLists.txt Makefile.am Makefile.inc config2setopts.c config2setopts.h curl.rc curlinfo.c mk-file-embed.pl mkhelp.pl slist_wc.c slist_wc.h terminal.c terminal.h tool_cb_dbg.c tool_cb_dbg.h tool_cb_hdr.c tool_cb_hdr.h tool_cb_prg.c tool_cb_prg.h tool_cb_rea.c tool_cb_rea.h tool_cb_see.c tool_cb_see.h tool_cb_soc.c tool_cb_soc.h tool_cb_wrt.c tool_cb_wrt.h tool_cfgable.c tool_cfgable.h tool_dirhie.c tool_dirhie.h tool_doswin.c tool_doswin.h tool_easysrc.c tool_easysrc.h tool_filetime.c tool_filetime.h tool_findfile.c tool_findfile.h tool_formparse.c tool_formparse.h tool_getparam.c tool_getparam.h tool_getpass.c tool_getpass.h tool_help.c tool_help.h tool_helpers.c tool_helpers.h tool_hugehelp.h tool_ipfs.c tool_ipfs.h tool_libinfo.c tool_libinfo.h tool_listhelp.c tool_main.c tool_main.h tool_msgs.c tool_msgs.h tool_operate.c tool_operate.h tool_operhlp.c tool_operhlp.h tool_paramhlp.c tool_paramhlp.h tool_parsecfg.c tool_parsecfg.h tool_progress.c tool_progress.h tool_sdecls.h tool_setopt.c tool_setopt.h tool_setup.h tool_ssls.c tool_ssls.h tool_stderr.c tool_stderr.h tool_urlglob.c tool_urlglob.h tool_util.c tool_util.h tool_version.h tool_vms.c tool_vms.h tool_writeout.c tool_writeout.h tool_writeout_json.c tool_writeout_json.h tool_xattr.c tool_xattr.h var.c var.h
tests
certs .gitignore CMakeLists.txt Makefile.am Makefile.inc genserv.pl srp-verifier-conf srp-verifier-db test-ca.cnf test-ca.prm test-client-cert.prm test-client-eku-only.prm test-localhost-san-first.prm test-localhost-san-last.prm test-localhost.nn.prm test-localhost.prm test-localhost0h.prm
cmake CMakeLists.txt test.c test.cpp test.sh
data .gitignore DISABLED Makefile.am data-xml1 data1400.c data1401.c data1402.c data1403.c data1404.c data1405.c data1406.c data1407.c data1420.c data1461.txt data1463.txt data1465.c data1481.c data1705-1.md data1705-2.md data1705-3.md data1705-4.md data1705-stdout.1 data1706-1.md data1706-2.md data1706-3.md data1706-4.md data1706-stdout.txt data320.html test1 test10 test100 test1000 test1001 test1002 test1003 test1004 test1005 test1006 test1007 test1008 test1009 test101 test1010 test1011 test1012 test1013 test1014 test1015 test1016 test1017 test1018 test1019 test102 test1020 test1021 test1022 test1023 test1024 test1025 test1026 test1027 test1028 test1029 test103 test1030 test1031 test1032 test1033 test1034 test1035 test1036 test1037 test1038 test1039 test104 test1040 test1041 test1042 test1043 test1044 test1045 test1046 test1047 test1048 test1049 test105 test1050 test1051 test1052 test1053 test1054 test1055 test1056 test1057 test1058 test1059 test106 test1060 test1061 test1062 test1063 test1064 test1065 test1066 test1067 test1068 test1069 test107 test1070 test1071 test1072 test1073 test1074 test1075 test1076 test1077 test1078 test1079 test108 test1080 test1081 test1082 test1083 test1084 test1085 test1086 test1087 test1088 test1089 test109 test1090 test1091 test1092 test1093 test1094 test1095 test1096 test1097 test1098 test1099 test11 test110 test1100 test1101 test1102 test1103 test1104 test1105 test1106 test1107 test1108 test1109 test111 test1110 test1111 test1112 test1113 test1114 test1115 test1116 test1117 test1118 test1119 test112 test1120 test1121 test1122 test1123 test1124 test1125 test1126 test1127 test1128 test1129 test113 test1130 test1131 test1132 test1133 test1134 test1135 test1136 test1137 test1138 test1139 test114 test1140 test1141 test1142 test1143 test1144 test1145 test1146 test1147 test1148 test1149 test115 test1150 test1151 test1152 test1153 test1154 test1155 test1156 test1157 test1158 test1159 test116 test1160 test1161 test1162 test1163 test1164 test1165 test1166 test1167 test1168 test1169 test117 test1170 test1171 test1172 test1173 test1174 test1175 test1176 test1177 test1178 test1179 test118 test1180 test1181 test1182 test1183 test1184 test1185 test1186 test1187 test1188 test1189 test119 test1190 test1191 test1192 test1193 test1194 test1195 test1196 test1197 test1198 test1199 test12 test120 test1200 test1201 test1202 test1203 test1204 test1205 test1206 test1207 test1208 test1209 test121 test1210 test1211 test1212 test1213 test1214 test1215 test1216 test1217 test1218 test1219 test122 test1220 test1221 test1222 test1223 test1224 test1225 test1226 test1227 test1228 test1229 test123 test1230 test1231 test1232 test1233 test1234 test1235 test1236 test1237 test1238 test1239 test124 test1240 test1241 test1242 test1243 test1244 test1245 test1246 test1247 test1248 test1249 test125 test1250 test1251 test1252 test1253 test1254 test1255 test1256 test1257 test1258 test1259 test126 test1260 test1261 test1262 test1263 test1264 test1265 test1266 test1267 test1268 test1269 test127 test1270 test1271 test1272 test1273 test1274 test1275 test1276 test1277 test1278 test1279 test128 test1280 test1281 test1282 test1283 test1284 test1285 test1286 test1287 test1288 test1289 test129 test1290 test1291 test1292 test1293 test1294 test1295 test1296 test1297 test1298 test1299 test13 test130 test1300 test1301 test1302 test1303 test1304 test1305 test1306 test1307 test1308 test1309 test131 test1310 test1311 test1312 test1313 test1314 test1315 test1316 test1317 test1318 test1319 test132 test1320 test1321 test1322 test1323 test1324 test1325 test1326 test1327 test1328 test1329 test133 test1330 test1331 test1332 test1333 test1334 test1335 test1336 test1337 test1338 test1339 test134 test1340 test1341 test1342 test1343 test1344 test1345 test1346 test1347 test1348 test1349 test135 test1350 test1351 test1352 test1353 test1354 test1355 test1356 test1357 test1358 test1359 test136 test1360 test1361 test1362 test1363 test1364 test1365 test1366 test1367 test1368 test1369 test137 test1370 test1371 test1372 test1373 test1374 test1375 test1376 test1377 test1378 test1379 test138 test1380 test1381 test1382 test1383 test1384 test1385 test1386 test1387 test1388 test1389 test139 test1390 test1391 test1392 test1393 test1394 test1395 test1396 test1397 test1398 test1399 test14 test140 test1400 test1401 test1402 test1403 test1404 test1405 test1406 test1407 test1408 test1409 test141 test1410 test1411 test1412 test1413 test1414 test1415 test1416 test1417 test1418 test1419 test142 test1420 test1421 test1422 test1423 test1424 test1425 test1426 test1427 test1428 test1429 test143 test1430 test1431 test1432 test1433 test1434 test1435 test1436 test1437 test1438 test1439 test144 test1440 test1441 test1442 test1443 test1444 test1445 test1446 test1447 test1448 test1449 test145 test1450 test1451 test1452 test1453 test1454 test1455 test1456 test1457 test1458 test1459 test146 test1460 test1461 test1462 test1463 test1464 test1465 test1466 test1467 test1468 test1469 test147 test1470 test1471 test1472 test1473 test1474 test1475 test1476 test1477 test1478 test1479 test148 test1480 test1481 test1482 test1483 test1484 test1485 test1486 test1487 test1488 test1489 test149 test1490 test1491 test1492 test1493 test1494 test1495 test1496 test1497 test1498 test1499 test15 test150 test1500 test1501 test1502 test1503 test1504 test1505 test1506 test1507 test1508 test1509 test151 test1510 test1511 test1512 test1513 test1514 test1515 test1516 test1517 test1518 test1519 test152 test1520 test1521 test1522 test1523 test1524 test1525 test1526 test1527 test1528 test1529 test153 test1530 test1531 test1532 test1533 test1534 test1535 test1536 test1537 test1538 test1539 test154 test1540 test1541 test1542 test1543 test1544 test1545 test1546 test1547 test1548 test1549 test155 test1550 test1551 test1552 test1553 test1554 test1555 test1556 test1557 test1558 test1559 test156 test1560 test1561 test1562 test1563 test1564 test1565 test1566 test1567 test1568 test1569 test157 test1570 test1571 test1572 test1573 test1574 test1575 test1576 test1577 test1578 test1579 test158 test1580 test1581 test1582 test1583 test1584 test1585 test1586 test1587 test1588 test1589 test159 test1590 test1591 test1592 test1593 test1594 test1595 test1596 test1597 test1598 test1599 test16 test160 test1600 test1601 test1602 test1603 test1604 test1605 test1606 test1607 test1608 test1609 test161 test1610 test1611 test1612 test1613 test1614 test1615 test1616 test1617 test1618 test1619 test162 test1620 test1621 test1622 test1623 test1624 test1625 test1626 test1627 test1628 test1629 test163 test1630 test1631 test1632 test1633 test1634 test1635 test1636 test1637 test1638 test1639 test164 test1640 test1641 test1642 test1643 test1644 test1645 test165 test1650 test1651 test1652 test1653 test1654 test1655 test1656 test1657 test1658 test1659 test166 test1660 test1661 test1662 test1663 test1664 test1665 test1666 test1667 test1668 test1669 test167 test1670 test1671 test1672 test1673 test1674 test1675 test1676 test168 test1680 test1681 test1682 test1683 test1684 test1685 test169 test17 test170 test1700 test1701 test1702 test1703 test1704 test1705 test1706 test1707 test1708 test1709 test171 test1710 test1711 test1712 test1713 test1714 test1715 test172 test1720 test1721 test173 test174 test175 test176 test177 test178 test179 test18 test180 test1800 test1801 test1802 test181 test182 test183 test184 test1847 test1848 test1849 test185 test1850 test1851 test186 test187 test188 test189 test19 test190 test1900 test1901 test1902 test1903 test1904 test1905 test1906 test1907 test1908 test1909 test191 test1910 test1911 test1912 test1913 test1914 test1915 test1916 test1917 test1918 test1919 test192 test1920 test1921 test193 test1933 test1934 test1935 test1936 test1937 test1938 test1939 test194 test1940 test1941 test1942 test1943 test1944 test1945 test1946 test1947 test1948 test195 test1955 test1956 test1957 test1958 test1959 test196 test1960 test1964 test1965 test1966 test197 test1970 test1971 test1972 test1973 test1974 test1975 test1976 test1977 test1978 test1979 test198 test1980 test1981 test1982 test1983 test1984 test199 test2 test20 test200 test2000 test2001 test2002 test2003 test2004 test2005 test2006 test2007 test2008 test2009 test201 test2010 test2011 test2012 test2013 test2014 test202 test2023 test2024 test2025 test2026 test2027 test2028 test2029 test203 test2030 test2031 test2032 test2033 test2034 test2035 test2037 test2038 test2039 test204 test2040 test2041 test2042 test2043 test2044 test2045 test2046 test2047 test2048 test2049 test205 test2050 test2051 test2052 test2053 test2054 test2055 test2056 test2057 test2058 test2059 test206 test2060 test2061 test2062 test2063 test2064 test2065 test2066 test2067 test2068 test2069 test207 test2070 test2071 test2072 test2073 test2074 test2075 test2076 test2077 test2078 test2079 test208 test2080 test2081 test2082 test2083 test2084 test2085 test2086 test2087 test2088 test2089 test209 test2090 test2091 test2092 test21 test210 test2100 test2101 test2102 test2103 test2104 test211 test212 test213 test214 test215 test216 test217 test218 test219 test22 test220 test2200 test2201 test2202 test2203 test2204 test2205 test2206 test2207 test221 test222 test223 test224 test225 test226 test227 test228 test229 test23 test230 test2300 test2301 test2302 test2303 test2304 test2306 test2307 test2308 test2309 test231 test232 test233 test234 test235 test236 test237 test238 test239 test24 test240 test2400 test2401 test2402 test2403 test2404 test2405 test2406 test2407 test2408 test2409 test241 test2410 test2411 test242 test243 test244 test245 test246 test247 test248 test249 test25 test250 test2500 test2501 test2502 test2503 test2504 test2505 test2506 test251 test252 test253 test254 test255 test256 test257 test258 test259 test26 test260 test2600 test2601 test2602 test2603 test2604 test2605 test261 test262 test263 test264 test265 test266 test267 test268 test269 test27 test270 test2700 test2701 test2702 test2703 test2704 test2705 test2706 test2707 test2708 test2709 test271 test2710 test2711 test2712 test2713 test2714 test2715 test2716 test2717 test2718 test2719 test272 test2720 test2721 test2722 test2723 test273 test274 test275 test276 test277 test278 test279 test28 test280 test281 test282 test283 test284 test285 test286 test287 test288 test289 test29 test290 test291 test292 test293 test294 test295 test296 test297 test298 test299 test3 test30 test300 test3000 test3001 test3002 test3003 test3004 test3005 test3006 test3007 test3008 test3009 test301 test3010 test3011 test3012 test3013 test3014 test3015 test3016 test3017 test3018 test3019 test302 test3020 test3021 test3022 test3023 test3024 test3025 test3026 test3027 test3028 test3029 test303 test3030 test3031 test3032 test3033 test3034 test3035 test3036 test304 test305 test306 test307 test308 test309 test31 test310 test3100 test3101 test3102 test3103 test3104 test3105 test3106 test311 test312 test313 test314 test315 test316 test317 test318 test319 test32 test320 test3200 test3201 test3202 test3203 test3204 test3205 test3206 test3207 test3208 test3209 test321 test3210 test3211 test3212 test3213 test3214 test3215 test3216 test3217 test3218 test3219 test322 test3220 test323 test324 test325 test326 test327 test328 test329 test33 test330 test3300 test3301 test3302 test331 test332 test333 test334 test335 test336 test337 test338 test339 test34 test340 test341 test342 test343 test344 test345 test346 test347 test348 test349 test35 test350 test351 test352 test353 test354 test355 test356 test357 test358 test359 test36 test360 test361 test362 test363 test364 test365 test366 test367 test368 test369 test37 test370 test371 test372 test373 test374 test375 test376 test378 test379 test38 test380 test381 test383 test384 test385 test386 test387 test388 test389 test39 test390 test391 test392 test393 test394 test395 test396 test397 test398 test399 test4 test40 test400 test4000 test4001 test401 test402 test403 test404 test405 test406 test407 test408 test409 test41 test410 test411 test412 test413 test414 test415 test416 test417 test418 test419 test42 test420 test421 test422 test423 test424 test425 test426 test427 test428 test429 test43 test430 test431 test432 test433 test434 test435 test436 test437 test438 test439 test44 test440 test441 test442 test443 test444 test445 test446 test447 test448 test449 test45 test450 test451 test452 test453 test454 test455 test456 test457 test458 test459 test46 test460 test461 test462 test463 test467 test468 test469 test47 test470 test471 test472 test473 test474 test475 test476 test477 test478 test479 test48 test480 test481 test482 test483 test484 test485 test486 test487 test488 test489 test49 test490 test491 test492 test493 test494 test495 test496 test497 test498 test499 test5 test50 test500 test501 test502 test503 test504 test505 test506 test507 test508 test509 test51 test510 test511 test512 test513 test514 test515 test516 test517 test518 test519 test52 test520 test521 test522 test523 test524 test525 test526 test527 test528 test529 test53 test530 test531 test532 test533 test534 test535 test536 test537 test538 test539 test54 test540 test541 test542 test543 test544 test545 test546 test547 test548 test549 test55 test550 test551 test552 test553 test554 test555 test556 test557 test558 test559 test56 test560 test561 test562 test563 test564 test565 test566 test567 test568 test569 test57 test570 test571 test572 test573 test574 test575 test576 test577 test578 test579 test58 test580 test581 test582 test583 test584 test585 test586 test587 test588 test589 test59 test590 test591 test592 test593 test594 test595 test596 test597 test598 test599 test6 test60 test600 test601 test602 test603 test604 test605 test606 test607 test608 test609 test61 test610 test611 test612 test613 test614 test615 test616 test617 test618 test619 test62 test620 test621 test622 test623 test624 test625 test626 test627 test628 test629 test63 test630 test631 test632 test633 test634 test635 test636 test637 test638 test639 test64 test640 test641 test642 test643 test644 test645 test646 test647 test648 test649 test65 test650 test651 test652 test653 test654 test655 test656 test658 test659 test66 test660 test661 test662 test663 test664 test665 test666 test667 test668 test669 test67 test670 test671 test672 test673 test674 test675 test676 test677 test678 test679 test68 test680 test681 test682 test683 test684 test685 test686 test687 test688 test689 test69 test690 test691 test692 test693 test694 test695 test696 test697 test698 test699 test7 test70 test700 test701 test702 test703 test704 test705 test706 test707 test708 test709 test71 test710 test711 test712 test713 test714 test715 test716 test717 test718 test719 test72 test720 test721 test722 test723 test724 test725 test726 test727 test728 test729 test73 test730 test731 test732 test733 test734 test735 test736 test737 test738 test739 test74 test740 test741 test742 test743 test744 test745 test746 test747 test748 test749 test75 test750 test751 test752 test753 test754 test755 test756 test757 test758 test759 test76 test760 test761 test762 test763 test764 test765 test766 test767 test768 test769 test77 test770 test771 test772 test773 test774 test775 test776 test777 test778 test779 test78 test780 test781 test782 test783 test784 test785 test786 test787 test788 test789 test79 test790 test791 test792 test793 test794 test795 test796 test797 test798 test799 test8 test80 test800 test801 test802 test803 test804 test805 test806 test807 test808 test809 test81 test810 test811 test812 test813 test814 test815 test816 test817 test818 test819 test82 test820 test821 test822 test823 test824 test825 test826 test827 test828 test829 test83 test830 test831 test832 test833 test834 test835 test836 test837 test838 test839 test84 test840 test841 test842 test843 test844 test845 test846 test847 test848 test849 test85 test850 test851 test852 test853 test854 test855 test856 test857 test858 test859 test86 test860 test861 test862 test863 test864 test865 test866 test867 test868 test869 test87 test870 test871 test872 test873 test874 test875 test876 test877 test878 test879 test88 test880 test881 test882 test883 test884 test885 test886 test887 test888 test889 test89 test890 test891 test892 test893 test894 test895 test896 test897 test898 test899 test9 test90 test900 test901 test902 test903 test904 test905 test906 test907 test908 test909 test91 test910 test911 test912 test913 test914 test915 test916 test917 test918 test919 test92 test920 test921 test922 test923 test924 test925 test926 test927 test928 test929 test93 test930 test931 test932 test933 test934 test935 test936 test937 test938 test939 test94 test940 test941 test942 test943 test944 test945 test946 test947 test948 test949 test95 test950 test951 test952 test953 test954 test955 test956 test957 test958 test959 test96 test960 test961 test962 test963 test964 test965 test966 test967 test968 test969 test97 test970 test971 test972 test973 test974 test975 test976 test977 test978 test979 test98 test980 test981 test982 test983 test984 test985 test986 test987 test988 test989 test99 test990 test991 test992 test993 test994 test995 test996 test997 test998 test999
http
testenv
mod_curltest .gitignore mod_curltest.c
__init__.py caddy.py certs.py client.py curl.py dante.py dnsd.py env.py httpd.py nghttpx.py ports.py sshd.py vsftpd.py ws_echo_server.py
.gitignore CMakeLists.txt Makefile.am config.ini.in conftest.py requirements.txt scorecard.py test_01_basic.py test_02_download.py test_03_goaway.py test_04_stuttered.py test_05_errors.py test_06_eyeballs.py test_07_upload.py test_08_caddy.py test_09_push.py test_10_proxy.py test_11_unix.py test_12_reuse.py test_13_proxy_auth.py test_14_auth.py test_15_tracing.py test_16_info.py test_17_ssl_use.py test_18_methods.py test_19_shutdown.py test_20_websockets.py test_21_resolve.py test_22_httpsrr.py test_30_vsftpd.py test_31_vsftpds.py test_32_ftps_vsftpd.py test_40_socks.py test_50_scp.py test_51_sftp.py
libtest .gitignore CMakeLists.txt Makefile.am Makefile.inc cli_ftp_upload.c cli_h2_pausing.c cli_h2_serverpush.c cli_h2_upgrade_extreme.c cli_hx_download.c cli_hx_upload.c cli_tls_session_reuse.c cli_upload_pausing.c cli_ws_data.c cli_ws_pingpong.c first.c first.h lib1156.c lib1301.c lib1308.c lib1485.c lib1500.c lib1501.c lib1502.c lib1506.c lib1507.c lib1508.c lib1509.c lib1510.c lib1511.c lib1512.c lib1513.c lib1514.c lib1515.c lib1517.c lib1518.c lib1520.c lib1522.c lib1523.c lib1525.c lib1526.c lib1527.c lib1528.c lib1529.c lib1530.c lib1531.c lib1532.c lib1533.c lib1534.c lib1535.c lib1536.c lib1537.c lib1538.c lib1540.c lib1541.c lib1542.c lib1545.c lib1549.c lib1550.c lib1551.c lib1552.c lib1553.c lib1554.c lib1555.c lib1556.c lib1557.c lib1558.c lib1559.c lib1560.c lib1564.c lib1565.c lib1567.c lib1568.c lib1569.c lib1571.c lib1576.c lib1582.c lib1587.c lib1588.c lib1589.c lib1591.c lib1592.c lib1593.c lib1594.c lib1597.c lib1598.c lib1599.c lib1662.c lib1900.c lib1901.c lib1902.c lib1903.c lib1905.c lib1906.c lib1907.c lib1908.c lib1910.c lib1911.c lib1912.c lib1913.c lib1915.c lib1916.c lib1918.c lib1919.c lib1920.c lib1921.c lib1933.c lib1934.c lib1935.c lib1936.c lib1937.c lib1938.c lib1939.c lib1940.c lib1945.c lib1947.c lib1948.c lib1955.c lib1956.c lib1957.c lib1958.c lib1959.c lib1960.c lib1964.c lib1965.c lib1970.c lib1971.c lib1972.c lib1973.c lib1974.c lib1975.c lib1977.c lib1978.c lib2023.c lib2032.c lib2082.c lib2301.c lib2302.c lib2304.c lib2306.c lib2308.c lib2309.c lib2402.c lib2404.c lib2405.c lib2502.c lib2504.c lib2505.c lib2506.c lib2700.c lib3010.c lib3025.c lib3026.c lib3027.c lib3033.c lib3034.c lib3100.c lib3101.c lib3102.c lib3103.c lib3104.c lib3105.c lib3207.c lib3208.c lib500.c lib501.c lib502.c lib503.c lib504.c lib505.c lib506.c lib507.c lib508.c lib509.c lib510.c lib511.c lib512.c lib513.c lib514.c lib515.c lib516.c lib517.c lib518.c lib519.c lib520.c lib521.c lib523.c lib524.c lib525.c lib526.c lib530.c lib533.c lib536.c lib537.c lib539.c lib540.c lib541.c lib542.c lib543.c lib544.c lib547.c lib549.c lib552.c lib553.c lib554.c lib555.c lib556.c lib557.c lib558.c lib559.c lib560.c lib562.c lib564.c lib566.c lib567.c lib568.c lib569.c lib570.c lib571.c lib572.c lib573.c lib574.c lib575.c lib576.c lib578.c lib579.c lib582.c lib583.c lib586.c lib589.c lib590.c lib591.c lib597.c lib598.c lib599.c lib643.c lib650.c lib651.c lib652.c lib653.c lib654.c lib655.c lib658.c lib659.c lib661.c lib666.c lib667.c lib668.c lib670.c lib674.c lib676.c lib677.c lib678.c lib694.c lib695.c lib751.c lib753.c lib757.c lib758.c lib766.c memptr.c mk-lib1521.pl test1013.pl test1022.pl test307.pl test610.pl test613.pl testtrace.c testtrace.h testutil.c testutil.h unitcheck.h
server .checksrc .gitignore CMakeLists.txt Makefile.am Makefile.inc dnsd.c first.c first.h getpart.c mqttd.c resolve.c rtspd.c sockfilt.c socksd.c sws.c tftpd.c util.c
tunit .gitignore CMakeLists.txt Makefile.am Makefile.inc README.md tool1394.c tool1604.c tool1621.c tool1622.c tool1623.c tool1720.c
unit .gitignore CMakeLists.txt Makefile.am Makefile.inc README.md unit1300.c unit1302.c unit1303.c unit1304.c unit1305.c unit1307.c unit1309.c unit1323.c unit1330.c unit1395.c unit1396.c unit1397.c unit1398.c unit1399.c unit1600.c unit1601.c unit1602.c unit1603.c unit1605.c unit1606.c unit1607.c unit1608.c unit1609.c unit1610.c unit1611.c unit1612.c unit1614.c unit1615.c unit1616.c unit1620.c unit1625.c unit1626.c unit1627.c unit1636.c unit1650.c unit1651.c unit1652.c unit1653.c unit1654.c unit1655.c unit1656.c unit1657.c unit1658.c unit1660.c unit1661.c unit1663.c unit1664.c unit1666.c unit1667.c unit1668.c unit1669.c unit1674.c unit1675.c unit1676.c unit1979.c unit1980.c unit2600.c unit2601.c unit2602.c unit2603.c unit2604.c unit2605.c unit3200.c unit3205.c unit3211.c unit3212.c unit3213.c unit3214.c unit3216.c unit3219.c unit3300.c unit3301.c unit3302.c
.gitignore CMakeLists.txt Makefile.am allversions.pm appveyor.pm azure.pm config.in configurehelp.pm.in devtest.pl dictserver.py directories.pm ech_combos.py ech_tests.sh ftpserver.pl getpart.pm globalconfig.pm http-server.pl http2-server.pl http3-server.pl memanalyze.pl memanalyzer.pm negtelnetserver.py nghttpx.conf pathhelp.pm processhelp.pm requirements.txt rtspserver.pl runner.pm runtests.pl secureserver.pl serverhelp.pm servers.pm smbserver.py sshhelp.pm sshserver.pl test1119.pl test1135.pl test1139.pl test1140.pl test1165.pl test1167.pl test1173.pl test1175.pl test1177.pl test1222.pl test1275.pl test1276.pl test1477.pl test1486.pl test1488.pl test1544.pl test1707.pl test745.pl test971.pl testcurl.pl testutil.pm tftpserver.pl util.py valgrind.pm valgrind.supp
.clang-tidy.yml .dir-locals.el .editorconfig .git-blame-ignore-revs .gitattributes .gitignore .mailmap CHANGES.md CMakeLists.txt COPYING Dockerfile GIT-INFO.md Makefile.am README README.md RELEASE-NOTES REUSE.toml SECURITY.md acinclude.m4 appveyor.sh appveyor.yml configure.ac curl-config.in libcurl.pc.in renovate.json
examples .env config.ini crypto_test.lua env_test.lua fs_example.lua http_server.lua https_test.lua ini_example.lua json.lua log.lua path_fs_example.lua process_example.lua request_download.lua request_test.lua run_all.lua sqlite_example.lua sqlite_http_template.lua stash_test.lua template_test.lua timer.lua websocket.lua
iniparser
.github
ISSUE_TEMPLATE config.yml
workflows disable-pull-requests.yml trigger-gitlab-ci.yml
cmake JoinPaths.cmake config.cmake.in pc.in
example iniexample.c iniwrite.c parse.c twisted-errors.ini twisted-genhuge.py twisted-ofkey.ini twisted-ofval.ini twisted.ini
src dictionary.c dictionary.h iniparser.c iniparser.h
test
ressources
bad_ini ends_well.ini twisted-errors.ini twisted-ofkey.ini twisted-ofval.ini
good_ini empty.ini spaced.ini spaced2.ini twisted.ini
gruezi.ini old.ini quotes.ini utf8.ini
CMakeLists.txt test_dictionary.c test_iniparser.c unity-config.yml unity_config.h
.cmake-format.py .gitignore .gitlab-ci.yml .gitmessage .travis.yml AUTHORS CMakeLists.txt FAQ-en.md FAQ-zhcn.md INSTALL LICENSE README.md compile_commands.json
jinjac
example CMakeLists.txt example.c
jinjac_test_app CMakeLists.txt jinjac_test_app.c
libjinjac
include jinjac.h
src CMakeLists.txt ast.c ast.h block_statement.c block_statement.h buffer.c buffer.h buildin.c buildin.h common.h convert.c convert.h flex_decl.h jfunction.c jfunction.h jinja_expression.l jinja_expression.y jinjac_parse.c jinjac_parse.h jinjac_stream.c jinjac_stream.h jlist.c jlist.h jobject.c jobject.h parameter.c parameter.h str_obj.c str_obj.h trace.c trace.h
CMakeLists.txt
test .gitignore CMakeLists.txt autotest.rb test_01.expected test_01.jinja test_01b.expected test_01b.jinja test_01c.expected test_01c.jinja test_01d.expected test_01d.jinja test_02.expected test_02.jinja test_03.expected test_03.jinja test_04.expected test_04.jinja test_05.expected test_05.jinja test_06.expected test_06.jinja test_07.expected test_07.jinja test_08.expected test_08.jinja test_08b.expected test_08b.jinja test_09.expected test_09.jinja test_10.expected test_10.jinja test_11.expected test_11.jinja test_12.expected test_12.jinja test_13.expected test_13.jinja test_14.expected test_14.jinja test_15.expected test_15.jinja test_16.expected test_16.jinja test_17.expected test_17.jinja test_18.expected test_18.jinja test_18b.expected test_18b.jinja test_18c.expected test_18c.jinja test_19.expected test_19.jinja test_19b.expected test_19b.jinja test_19c.expected test_19c.jinja test_19d.expected test_19d.jinja test_19e.expected test_19e.jinja test_19f.expected test_19f.jinja test_20.expected test_20.jinja test_21.expected test_21.jinja test_22.expected test_22.jinja test_22a.expected test_22a.jinja test_22b.expected test_22b.jinja test_23.expected test_23.jinja test_24.expected test_24.jinja
.gitignore CMakeLists.txt LICENSE.txt README.md build_coverage.sh build_debug.sh build_release.sh cppcheck_analysis.sh
libev Changes LICENSE Makefile Makefile.am Makefile.in README Symbols.ev Symbols.event aclocal.m4 autogen.sh compile config.guess config.h config.h.in config.status config.sub configure configure.ac depcomp ev++.h ev.3 ev.c ev.h ev.pod ev_epoll.c ev_kqueue.c ev_poll.c ev_port.c ev_select.c ev_vars.h ev_win32.c ev_wrap.h event.c event.h install-sh libev.m4 libtool ltmain.sh missing mkinstalldirs stamp-h1
luajit
doc
img contact.png
bluequad-print.css bluequad.css contact.html ext_buffer.html ext_c_api.html ext_ffi.html ext_ffi_api.html ext_ffi_semantics.html ext_ffi_tutorial.html ext_jit.html ext_profiler.html extensions.html install.html luajit.html running.html
dynasm dasm_arm.h dasm_arm.lua dasm_arm64.h dasm_arm64.lua dasm_mips.h dasm_mips.lua dasm_mips64.lua dasm_ppc.h dasm_ppc.lua dasm_proto.h dasm_x64.lua dasm_x86.h dasm_x86.lua dynasm.lua
etc luajit.1 luajit.pc
src
host .gitignore README buildvm.c buildvm.h buildvm_asm.c buildvm_fold.c buildvm_lib.c buildvm_libbc.h buildvm_peobj.c genlibbc.lua genminilua.lua genversion.lua minilua.c
jit .gitignore bc.lua bcsave.lua dis_arm.lua dis_arm64.lua dis_arm64be.lua dis_mips.lua dis_mips64.lua dis_mips64el.lua dis_mips64r6.lua dis_mips64r6el.lua dis_mipsel.lua dis_ppc.lua dis_x64.lua dis_x86.lua dump.lua p.lua v.lua zone.lua
.gitignore Makefile Makefile.dep lauxlib.h lib_aux.c lib_base.c lib_bit.c lib_buffer.c lib_debug.c lib_ffi.c lib_init.c lib_io.c lib_jit.c lib_math.c lib_os.c lib_package.c lib_string.c lib_table.c lj_alloc.c lj_alloc.h lj_api.c lj_arch.h lj_asm.c lj_asm.h lj_asm_arm.h lj_asm_arm64.h lj_asm_mips.h lj_asm_ppc.h lj_asm_x86.h lj_assert.c lj_bc.c lj_bc.h lj_bcdump.h lj_bcread.c lj_bcwrite.c lj_buf.c lj_buf.h lj_carith.c lj_carith.h lj_ccall.c lj_ccall.h lj_ccallback.c lj_ccallback.h lj_cconv.c lj_cconv.h lj_cdata.c lj_cdata.h lj_char.c lj_char.h lj_clib.c lj_clib.h lj_cparse.c lj_cparse.h lj_crecord.c lj_crecord.h lj_ctype.c lj_ctype.h lj_debug.c lj_debug.h lj_def.h lj_dispatch.c lj_dispatch.h lj_emit_arm.h lj_emit_arm64.h lj_emit_mips.h lj_emit_ppc.h lj_emit_x86.h lj_err.c lj_err.h lj_errmsg.h lj_ff.h lj_ffrecord.c lj_ffrecord.h lj_frame.h lj_func.c lj_func.h lj_gc.c lj_gc.h lj_gdbjit.c lj_gdbjit.h lj_ir.c lj_ir.h lj_ircall.h lj_iropt.h lj_jit.h lj_lex.c lj_lex.h lj_lib.c lj_lib.h lj_load.c lj_mcode.c lj_mcode.h lj_meta.c lj_meta.h lj_obj.c lj_obj.h lj_opt_dce.c lj_opt_fold.c lj_opt_loop.c lj_opt_mem.c lj_opt_narrow.c lj_opt_sink.c lj_opt_split.c lj_parse.c lj_parse.h lj_prng.c lj_prng.h lj_profile.c lj_profile.h lj_record.c lj_record.h lj_serialize.c lj_serialize.h lj_snap.c lj_snap.h lj_state.c lj_state.h lj_str.c lj_str.h lj_strfmt.c lj_strfmt.h lj_strfmt_num.c lj_strscan.c lj_strscan.h lj_tab.c lj_tab.h lj_target.h lj_target_arm.h lj_target_arm64.h lj_target_mips.h lj_target_ppc.h lj_target_x86.h lj_trace.c lj_trace.h lj_traceerr.h lj_udata.c lj_udata.h lj_vm.h lj_vmevent.c lj_vmevent.h lj_vmmath.c ljamalg.c lua.h lua.hpp luaconf.h luajit.c luajit_rolling.h lualib.h msvcbuild.bat nxbuild.bat ps4build.bat ps5build.bat psvitabuild.bat vm_arm.dasc vm_arm64.dasc vm_mips.dasc vm_mips64.dasc vm_ppc.dasc vm_x64.dasc vm_x86.dasc xb1build.bat xedkbuild.bat
.gitattributes .gitignore .relver COPYRIGHT Makefile README
sqlite shell.c sqlite3.c sqlite3.h sqlite3ext.h
wolfssl
.github
ISSUE_TEMPLATE bug_report.yaml other.yaml
actions
install-apt-deps action.yml
scripts
zephyr-4.x external_libc.conf zephyr-test.sh
openssl-ech.sh tls-anvil-test.sh
workflows
disabled haproxy.yml hitch.yml hostap.yml
hostap-files
configs
07c9f183ea744ac04585fb6dd10220c75a5e2e74 hostapd.config tests wpa_supplicant.config
b607d2723e927a3446d89aed813f1aa6068186bb hostapd.config tests wpa_supplicant.config
hostap_2_10 extra.patch hostapd.config tests wpa_supplicant.config
Makefile README dbus-wpa_supplicant.conf
ada.yml arduino.yml async-examples.yml async.yml atecc608-sim.yml bind.yml cmake-autoconf.yml cmake.yml codespell.yml coverity-scan-fixes.yml cryptocb-only.yml curl.yml cyrus-sasl.yml disable-pk-algs.yml docker-Espressif.yml docker-OpenWrt.yml emnet-nonblock.yml fil-c.yml freertos-mem-track.yml gencertbuf.yml grpc.yml haproxy.yml hostap-vm.yml intelasm-c-fallback.yml ipmitool.yml jwt-cpp.yml krb5.yml libspdm.yml libssh2.yml libvncserver.yml linuxkm.yml macos-apple-native-cert-validation.yml mbedtls.sh mbedtls.yml membrowse-comment.yml membrowse-onboard.yml membrowse-report.yml memcached.sh memcached.yml mono.yml mosquitto.yml msmtp.yml msys2.yml multi-arch.yml multi-compiler.yml net-snmp.yml nginx.yml no-malloc.yml no-tls.yml nss.sh nss.yml ntp.yml ocsp.yml openldap.yml openssh.yml openssl-ech.yml opensslcoexist.yml openvpn.yml os-check.yml packaging.yml pam-ipmi.yml pq-all.yml pr-commit-check.yml psk.yml puf.yml python.yml rng-tools.yml rust-wrapper.yml se050-sim.yml smallStackSize.yml socat.yml softhsm.yml sssd.yml stm32-sim.yml stsafe-a120-sim.yml stunnel.yml symbol-prefixes.yml threadx.yml tls-anvil.yml trackmemory.yml watcomc.yml win-csharp-test.yml wolfCrypt-Wconversion.yml wolfboot-integration.yml wolfsm.yml xcode.yml zephyr-4.x.yml zephyr.yml
PULL_REQUEST_TEMPLATE.md SECURITY.md membrowse-targets.json
Docker
OpenWrt Dockerfile README.md runTests.sh
packaging
debian Dockerfile
fedora Dockerfile
wolfCLU Dockerfile README.md
yocto Dockerfile buildAndPush.sh
Dockerfile Dockerfile.cross-compiler README.md buildAndPush.sh include.am run.sh
IDE
ARDUINO
sketches
wolfssl_client README.md
wolfssl_server README.md
wolfssl_version README.md
README.md
Arduino_README_prepend.md README.md include.am keywords.txt library.properties.template wolfssl-arduino.cpp wolfssl-arduino.sh wolfssl.h
AURIX Cpu0_Main.c README.md include.am user_settings.h wolf_main.c
Android Android.bp README.md include.am user_settings.h
CRYPTOCELL README.md include.am main.c user_settings.h
CSBENCH include.am user_settings.h
ECLIPSE
DEOS
deos_wolfssl .options
README.md deos_malloc.c include.am tls_wolfssl.c tls_wolfssl.h user_settings.h
MICRIUM README.md client_wolfssl.c client_wolfssl.h include.am server_wolfssl.c server_wolfssl.h user_settings.h wolfsslRunTests.c
RTTHREAD README.md include.am user_settings.h wolfssl_test.c
SIFIVE README.md include.am
Espressif
ESP-IDF
examples
template
VisualGDB wolfssl_template_IDF_v5.1_ESP32.vgdbproj
components
wolfssl
include user_settings.h
CMakeLists.txt Kconfig README.md component.mk
main
include main.h
CMakeLists.txt Kconfig.projbuild component.mk main.c
CMakeLists.txt Makefile README.md partitions_singleapp_large.csv sdkconfig.defaults sdkconfig.defaults.esp8266
wolfssl_benchmark
VisualGDB wolfssl_benchmark_IDF_v4.4_ESP32.sln wolfssl_benchmark_IDF_v4.4_ESP32.vgdbproj wolfssl_benchmark_IDF_v5_ESP32.sln wolfssl_benchmark_IDF_v5_ESP32.vgdbproj wolfssl_benchmark_IDF_v5_ESP32C3.sln wolfssl_benchmark_IDF_v5_ESP32C3.vgdbproj wolfssl_benchmark_IDF_v5_ESP32S3.sln wolfssl_benchmark_IDF_v5_ESP32S3.vgdbproj
components
wolfssl
include user_settings.h
CMakeLists.txt Kconfig README.md component.mk
main
include main.h
CMakeLists.txt Kconfig.projbuild component.mk main.c
CMakeLists.txt Makefile README.md partitions_singleapp_large.csv sdkconfig.defaults sdkconfig.defaults.esp8266
wolfssl_client
VisualGDB README.md wolfssl_client_IDF_v5_ESP32.sln wolfssl_client_IDF_v5_ESP32.vgdbproj
components
wolfssl
include user_settings.h
CMakeLists.txt Kconfig README.md component.mk
main
include client-tls.h main.h time_helper.h wifi_connect.h
CMakeLists.txt Kconfig.projbuild client-tls.c component.mk main.c time_helper.c wifi_connect.c
CMakeLists.txt Makefile README.md README_server_sm.md partitions_singleapp_large.csv sdkconfig.defaults sdkconfig.defaults.esp32c2 sdkconfig.defaults.esp8266 wolfssl_client_ESP8266.vgdbproj
wolfssl_server
VisualGDB README.md wolfssl_server_IDF_v5_ESP32.sln wolfssl_server_IDF_v5_ESP32.vgdbproj
components
wolfssl
include user_settings.h
CMakeLists.txt Kconfig README.md component.mk
main
include main.h server-tls.h time_helper.h wifi_connect.h
CMakeLists.txt Kconfig.projbuild component.mk main.c server-tls.c time_helper.c wifi_connect.c
CMakeLists.txt Makefile README.md README_server_sm.md partitions_singleapp_large.csv sdkconfig.defaults sdkconfig.defaults.esp32c2 sdkconfig.defaults.esp8266 wolfssl_server_ESP8266.vgdbproj
wolfssl_test
VisualGDB wolfssl_test-IDF_v5_ESP32.sln wolfssl_test-IDF_v5_ESP32.vgdbproj wolfssl_test-IDF_v5_ESP32C3.sln wolfssl_test-IDF_v5_ESP32C3.vgdbproj wolfssl_test-IDF_v5_ESP32C6.sln wolfssl_test-IDF_v5_ESP32C6.vgdbproj wolfssl_test_IDF_v5_ESP32S3.sln wolfssl_test_IDF_v5_ESP32S3.vgdbproj
components
wolfssl
include user_settings.h
CMakeLists.txt Kconfig README.md component.mk
main
include main.h
CMakeLists.txt Kconfig.projbuild component.mk main.c
CMakeLists.txt Makefile README.md partitions_singleapp_large.csv sdkconfig.defaults sdkconfig.defaults.esp32 sdkconfig.defaults.esp32c3 sdkconfig.defaults.esp32c6 sdkconfig.defaults.esp32h2 sdkconfig.defaults.esp32s2 sdkconfig.defaults.esp32s3 sdkconfig.defaults.esp8266 testAll.sh testMonitor.sh wolfssl_test_ESP8266.sln wolfssl_test_ESP8266.vgdbproj
wolfssl_test_idf
VisualGDB VisualGDB_wolfssl_test_idf.sln VisualGDB_wolfssl_test_idf.vgdbproj
main CMakeLists.txt Kconfig.projbuild component.mk main.c main_wip.c.ex time_helper.c time_helper.h
CMakeLists.txt Kconfig.projbuild README.md component.mk sdkconfig.defaults
README.md
libs CMakeLists.txt README.md component.mk tigard.cfg
test CMakeLists.txt README.md component.mk test_wolfssl.c
README.md README_32se.md UPDATE.md compileAllExamples.sh dummy_config_h dummy_test_paths.h setup.sh setup_win.bat user_settings.h
README.md include.am
GCC-ARM
Header user_settings.h
Source armtarget.c benchmark_main.c test_main.c tls_client.c tls_server.c wolf_main.c
Makefile Makefile.bench Makefile.client Makefile.common Makefile.server Makefile.static Makefile.test README.md include.am linker.ld linker_fips.ld
Gaisler-BCC README.md include.am
HEXAGON
DSP Makefile wolfssl_dsp.idl
Makefile README.md build.sh ecc-verify-benchmark.c ecc-verify.c include.am user_settings.h
HEXIWEAR
wolfSSL_HW .cwGeneratedFileSetLog user_settings.h
IAR-EWARM
Projects
benchmark benchmark-main.c current_time.c wolfCrypt-benchmark.ewd wolfCrypt-benchmark.ewp
common minimum-startup.c wolfssl.icf
lib wolfSSL-Lib.ewd wolfSSL-Lib.ewp
test test-main.c wolfCrypt-test.ewd wolfCrypt-test.ewp
user_settings.h wolfssl.eww
embOS
SAMV71_XULT
embOS_SAMV71_XULT_Linker_Script samv71q21_wolfssl.icf
embOS_SAMV71_XULT_user_settings user_settings.h user_settings_simple_example.h user_settings_verbose_example.h
embOS_wolfcrypt_benchmark_SAMV71_XULT
Application runBenchmarks.c
README_wolfcrypt_benchmark wolfcrypt_benchmark.ewd wolfcrypt_benchmark.ewp
embOS_wolfcrypt_lib_SAMV71_XULT README_wolfcrypt_lib wolfcrypt_lib.ewd wolfcrypt_lib.ewp
embOS_wolfcrypt_test_SAMV71_XULT
Application runWolfcryptTests.c
README_wolfcrypt_test wolfcrypt_test.ewd wolfcrypt_test.ewp
README_SAMV71
custom_port
custom_port_Linker_Script samv71q21_wolfssl.icf
custom_port_user_settings user_settings.h
wolfcrypt_benchmark_custom_port
Application runBenchmarks.c
wolfcrypt_test_custom_port
Application runWolfcryptTests.c
README_custom_port
extract_trial_here README_extract_trial_here
README
.gitignore README
IAR-MSP430 Makefile README.md include.am main.c user_settings.h
INTIME-RTOS Makefile README.md include.am libwolfssl.c libwolfssl.vcxproj user_settings.h wolfExamples.c wolfExamples.h wolfExamples.sln wolfExamples.vcxproj wolfssl-lib.sln wolfssl-lib.vcxproj
Infineon README.md include.am user_settings.h
KDS
config user_settings.h
include.am
LINUX-SGX README.md build.sh clean.sh include.am sgx_t_static.mk
LPCXPRESSO
lib_wolfssl lpc_18xx_port.c user_settings.h
wolf_example
src lpc_18xx_startup.c wolfssl_example.c
readme.txt
README.md
M68K
benchmark Makefile main.cpp
testwolfcrypt Makefile main.cpp
Makefile README.md include.am user_settings.h
MCUEXPRESSO
RT1170 fsl_caam_c.patch fsl_caam_h.patch user_settings.h
benchmark
source run_benchmark.c
wolfssl liblinks.xml
README.md include.am user_settings.h wolfcrypt_test.c
MDK-ARM
LPC43xx time-LCP43xx.c
MDK-ARM
wolfSSL Retarget.c cert_data.c cert_data.h config-BARE-METAL.h config-FS.h config-RTX-TCP-FS.h config-WOLFLIB.h main.c shell.c time-CortexM3-4.c time-dummy.c wolfssl_MDK_ARM.c wolfssl_MDK_ARM.h
STM32F2xx_StdPeriph_Lib time-STM32F2xx.c
MDK5-ARM
Conf user_settings.h
Inc wolfssl_MDK_ARM.h
Projects
CryptBenchmark Abstract.txt CryptBenchmark.sct CryptBenchmark.uvoptx CryptBenchmark.uvprojx main.c
CryptTest Abstract.txt CryptTest.sct CryptTest.uvoptx CryptTest.uvprojx main.c
EchoClient Abstract.txt EchoClient.uvoptx EchoClient.uvprojx main.c wolfssl-link.sct
EchoServer Abstract.txt EchoServer.uvoptx EchoServer.uvprojx main.c wolfssl-link.sct
SimpleClient Abstract.txt SimpleClient.uvoptx SimpleClient.uvprojx main.c wolfssl-link.sct
SimpleServer Abstract.txt SimpleServer.uvoptx SimpleServer.uvprojx main.c wolfssl-link.sct
wolfSSL-Full Abstract.txt main.c shell.c time-CortexM3-4.c wolfsslFull.uvoptx wolfsslFull.uvprojx
wolfSSL-Lib Abstract.txt wolfSSL-Lib.uvoptx wolfSSL-Lib.uvprojx
Src ssl-dummy.c
README.md include.am
MPLABX16
wolfcrypt_test.X
nbproject
private configurations.xml private.xml
configurations.xml include.am project.xml
Makefile
wolfssl.X
nbproject configurations.xml include.am project.xml
Makefile
README.md include.am main.c user_settings.h
MQX Makefile README-jp.md README.md client-tls.c include.am server-tls.c user_config.h user_settings.h
MSVS-2019-AZSPHERE
client client.c client.h
server server.c server.h
shared util.h
wolfssl_new_azsphere
HardwareDefinitions
avnet_mt3620_sk
inc
hw template_appliance.h
template_appliance.json
mt3620_rdb
inc
hw template_appliance.h
template_appliance.json
seeed_mt3620_mdb
inc
hw template_appliance.h
template_appliance.json
.gitignore CMakeLists.txt CMakeSettings.json app_manifest.json applibs_versions.h launch.vs.json main.c
README.md include.am user_settings.h
MYSQL CMakeLists_wolfCrypt.txt CMakeLists_wolfSSL.txt do.sh
NDS README.md
NETOS Makefile.wolfcrypt.inc README.md include.am user_settings.h user_settings.h-cert2425 user_settings.h-cert3389 wolfssl_netos_custom.c
OPENSTM32 README.md
PlatformIO
examples
wolfssl_benchmark
include README main.h
lib README
src CMakeLists.txt main.c
test README
CMakeLists.txt README.md platformio.ini sdkconfig.defaults wolfssl_benchmark.code-workspace
wolfssl_test
include README main.h
lib README
src CMakeLists.txt main.c
test README
CMakeLists.txt README.md platformio.ini sdkconfig.defaults wolfssl_test.code-workspace
README.md wolfssl_platformio.code-workspace
README.md include.am
QNX
CAAM-DRIVER Makefile
example-client Makefile client-tls.c
example-cmac Makefile cmac-test.c
example-server Makefile server-tls.c
testwolfcrypt Makefile
wolfssl Makefile user_settings.h
README.md include.am
RISCV
SIFIVE-HIFIVE1 Makefile README.md include.am main.c user_settings.h
SIFIVE-UNLEASHED README.md include.am
include.am
ROWLEY-CROSSWORKS-ARM Kinetis_FlashPlacement.xml README.md arm_startup.c benchmark_main.c hw.h include.am kinetis_hw.c retarget.c test_main.c user_settings.h wolfssl.hzp wolfssl_ltc.hzp
Renesas
cs+
Projects
common strings.h unistd.h user_settings.h wolfssl_dummy.c
t4_demo README_en.txt README_jp.txt t4_demo.mtpj wolf_client.c wolf_demo.h wolf_main.c wolf_server.c
test test.mtpj test_main.c
wolfssl_lib wolfssl_lib.mtpj
README include.am
e2studio
DK-S7G2
benchmark-template
src app_entry.c
example_server-template
src app_entry.c
wolfcrypttest-template
src app_entry.c
wolfssl-template-project configuration.xml
README.md include.am user_settings.h
Projects
common strings.h unistd.h user_settings.h wolfssl_dummy.c
test
src key_data.c key_data.h test_main.c wolf_client.c wolf_server.c wolfssl_demo.h
tools generate_rsa_keypair.sh genhexbuf.pl rsa_pss_sign.sh
wolfssl
src .gitkeep
wolfcrypt
src .gitkeep
README include.am
RA6M3
benchmark-wolfcrypt
common .gitkeep
script .gitkeep
src wolfssl_thread_entry.c
client-wolfssl
common
src .gitkeep
script .gitkeep
src wolfssl_thread_entry.c
wolfssl_thread_entry.h
common
ra6m3g README.md
src freertos_tcp_port.c
user_settings.h util.h
server-wolfssl
common
src .gitkeep
script .gitkeep
src wolfssl_thread_entry.c
wolfssl_thread_entry.h
test-wolfcrypt
common .gitkeep
script .gitkeep
src wolfssl_thread_entry.c
wolfssl
src .gitkeep
wolfcrypt .gitkeep
README.md README_APRA6M_en.md README_APRA6M_jp.md include.am
RA6M3G README.md
RA6M4
common user_settings.h wolfssl_demo.h
test
key_data key_data.h key_data_sce.c
src
SEGGER_RTT myprint.c
common .gitignore
test_main.c wolf_client.c wolfssl_sce_unit_test.c
test_RA6M4Debug.launch
tools
example_keys generate_SignedCA.sh rsa_private.pem rsa_public.pem
README.md
README.md include.am
RX65N
GR-ROSE
common strings.h unistd.h user_settings.h wolfssl_dummy.c
smc smc.scfg
test
src key_data.c key_data.h test_main.c wolf_client.c wolf_server.c wolfssl_demo.h
test.rcpc test_HardwareDebug.launch
tools
example_keys generate_SignedCA.sh rsa_private.pem rsa_public.pem
README.md
wolfssl wolfssl.rcpc
README_EN.md README_JP.md include.am
RSK
resource section.esi
wolfssl wolfssl.rcpc
wolfssl_demo key_data.c key_data.h user_settings.h wolfssl_demo.c wolfssl_demo.h
InstructionManualForExample_RSK+RX65N-2MB_EN.pdf InstructionManualForExample_RSK+RX65N-2MB_JP.pdf README_EN.md README_JP.md include.am
RX72N
EnvisionKit
Simple
common sectioninfo.esi wolfssl_dummy.c
test
src
client simple_tcp_client.c simple_tls_tsip_client.c
server simple_tcp_server.c simple_tls_server.c
test_main.c wolfssl_simple_demo.h
test.rcpc test.scfg test_HardwareDebug.launch
wolfssl wolfssl.rcpc
README_EN.md README_JP.md
resource section.esi
tools
example_keys generate_SignedCA.sh rsa_private.pem rsa_public.pem
README.md
wolfssl wolfssl.rcpc
wolfssl_demo key_data.c key_data.h user_settings.h wolfssl_demo.c wolfssl_demo.h wolfssl_tsip_unit_test.c
InstructionManualForExample_RX72N_EnvisonKit_EN.pdf InstructionManualForExample_RX72N_EnvisonKit_JP.pdf README_EN.md README_JP.md include.am
RZN2L
common user_settings.h wolfssl_demo.h
test
src
serial_io app_print.c
test wolf_client.c wolf_server.c wolfssl_rsip_unit_test.c
wolfCrypt .gitignore
wolfSSL .gitignore
local_system_init.c rzn2l_tst_thread_entry.c wolfssl_dummy.c
README.md include.am
SK-S7G2
common user_settings.h
wolfssl_lib configuration.xml
.gitignore README.md include.am
STARCORE README.txt include.am starcore_test.c user_settings.h
STM32Cube README.md STM32_Benchmarks.md default_conf.ftl include.am main.c wolfssl_example.c wolfssl_example.h
SimplicityStudio README.md include.am test_wolf.c user_settings.h
TRUESTUDIO
wolfssl user_settings.h
README include.am
VS-ARM README.md include.am user_settings.h wolfssl.sln wolfssl.vcxproj
VS-AZURE-SPHERE
client app_manifest.json client.c client.h client.vcxproj
server app_manifest.json server.c server.h server.vcxproj
shared util.h
wolfcrypt_test app_manifest.json wolfcrypt_test.vcxproj
README.md include.am user_settings.h wolfssl.sln wolfssl.vcxproj
VisualDSP include.am user_settings.h wolf_tasks.c
WICED-STUDIO README include.am user_settings.h
WIN README.txt include.am test.vcxproj user_settings.h user_settings_dtls.h wolfssl-fips.sln wolfssl-fips.vcxproj
WIN-SGX ReadMe.txt include.am wolfSSL_SGX.edl wolfSSL_SGX.sln wolfSSL_SGX.vcxproj
WIN-SRTP-KDF-140-3 README.txt include.am resource.h test.vcxproj user_settings.h wolfssl-fips.rc wolfssl-fips.sln wolfssl-fips.vcxproj
WIN10 README.txt include.am resource.h test.vcxproj user_settings.h wolfssl-fips.rc wolfssl-fips.sln wolfssl-fips.vcxproj
WINCE README.md include.am user_settings.h user_settings.h.140-2-deprecated
WORKBENCH README.md include.am
XCODE
Benchmark
wolfBench
Assets.xcassets
AppIcon.appiconset Contents.json
Base.lproj LaunchScreen.storyboard Main.storyboard
AppDelegate.h AppDelegate.m Info.plist ViewController.h ViewController.m main.m
wolfBench.xcodeproj project.pbxproj
include.am
wolfssl-FIPS.xcodeproj project.pbxproj
wolfssl.xcodeproj project.pbxproj
wolfssl_testsuite.xcodeproj project.pbxproj
README.md build-for-i386.sh include.am user_settings.h
XCODE-FIPSv2
macOS-C++
Intel user_settings.h
M1 user_settings.h
include.am user_settings.h
XCODE-FIPSv5 README include.am user_settings.h
XCODE-FIPSv6 README include.am user_settings.h
XilinxSDK
2018_2 lscript.ld
2019_2
wolfCrypt_example
src lscript.ld
wolfCrypt_example_system wolfCrypt_example_system.sprj
2022_1
wolfCrypt_FreeRTOS_example wolfCrypt_FreeRTOS_example.prj
wolfCrypt_FreeRTOS_example_system wolfCrypt_FreeRTOS_example_system.sprj
wolfCrypt_example wolfCrypt_example.prj
wolfCrypt_example_system wolfCrypt_example_system.sprj
.gitignore
README.md bench.sh combine.sh eclipse_formatter_profile.xml graph.sh include.am user_settings.h wolfssl_example.c
apple-universal
wolfssl-multiplatform
wolfssl-multiplatform
Assets.xcassets
AccentColor.colorset Contents.json
AppIcon.appiconset Contents.json
Contents.json
ContentView.swift simple_client_example.c simple_client_example.h wolfssl-multiplatform-Bridging-Header.h wolfssl_multiplatform.entitlements wolfssl_multiplatformApp.swift wolfssl_test_driver.c wolfssl_test_driver.h
wolfssl-multiplatform.xcodeproj project.pbxproj
.gitignore README.md build-wolfssl-framework.sh include.am
iotsafe Makefile README.md ca-cert.c devices.c devices.h include.am main.c memory-tls.c startup.c target.ld user_settings.h
iotsafe-raspberrypi Makefile README.md client-tls13.c include.am main.c
mynewt README.md apps.wolfcrypttest.pkg.yml crypto.wolfssl.pkg.yml crypto.wolfssl.syscfg.yml include.am setup.sh
zephyr README.md include.am
include.am
RTOS
nuttx
wolfssl .gitignore Kconfig Make.defs Makefile README.md setup-wolfssl.sh user_settings.h
include.am
bsdkm Makefile README.md bsdkm_wc_port.h include.am wolfkmod.c wolfkmod_aes.c x86_vecreg.c
certs
1024 ca-cert.der ca-cert.pem ca-key.der ca-key.pem client-cert.der client-cert.pem client-key.der client-key.pem client-keyPub.der dh1024.der dh1024.pem dsa-pub-1024.pem dsa1024.der dsa1024.pem include.am rsa1024.der server-cert.der server-cert.pem server-key.der server-key.pem
3072 client-cert.der client-cert.pem client-key.der client-key.pem client-keyPub.der include.am
4096 client-cert.der client-cert.pem client-key.der client-key.pem client-keyPub.der include.am
acert
rsa_pss acert.pem acert_ietf.pem acert_ietf_pubkey.pem acert_pubkey.pem
acert.pem acert_ietf.pem acert_ietf_pubkey.pem acert_pubkey.pem include.am
aia ca-issuers-cert.pem multi-aia-cert.pem overflow-aia-cert.pem
crl
extra-crls ca-int-cert-revoked.pem claim-root.pem crl_critical_entry.pem crlnum_57oct.pem crlnum_64oct.pem general-server-crl.pem large_crlnum.pem large_crlnum2.pem
hash_der 0fdb2da4.r0
hash_pem 0fdb2da4.r0
bad_time_fmt.pem ca-int-ecc.pem ca-int.pem ca-int2-ecc.pem ca-int2.pem caEcc384Crl.pem caEccCrl.der caEccCrl.pem cliCrl.pem client-int-ecc.pem client-int.pem crl.der crl.pem crl.revoked crl2.der crl2.pem crl_reason.pem crl_rsapss.pem eccCliCRL.pem eccSrvCRL.pem gencrls.sh include.am server-goodaltCrl.pem server-goodaltwildCrl.pem server-goodcnCrl.pem server-goodcnwildCrl.pem server-int-ecc.pem server-int.pem wolfssl.cnf
dilithium bench_dilithium_level2_key.der bench_dilithium_level3_key.der bench_dilithium_level5_key.der include.am
ecc bp256r1-key.der bp256r1-key.pem ca-secp256k1-cert.pem ca-secp256k1-key.pem client-bp256r1-cert.der client-bp256r1-cert.pem client-secp256k1-cert.der client-secp256k1-cert.pem genecc.sh include.am secp256k1-key.der secp256k1-key.pem secp256k1-param.pem secp256k1-privkey.der secp256k1-privkey.pem server-bp256r1-cert.der server-bp256r1-cert.pem server-secp256k1-cert.der server-secp256k1-cert.pem server2-secp256k1-cert.der server2-secp256k1-cert.pem wolfssl.cnf wolfssl_384.cnf
ed25519 ca-ed25519-key.der ca-ed25519-key.pem ca-ed25519-priv.der ca-ed25519-priv.pem ca-ed25519.der ca-ed25519.pem client-ed25519-key.der client-ed25519-key.pem client-ed25519-priv.der client-ed25519-priv.pem client-ed25519.der client-ed25519.pem eddsa-ed25519.der eddsa-ed25519.pem gen-ed25519-certs.sh gen-ed25519-keys.sh gen-ed25519.sh include.am root-ed25519-key.der root-ed25519-key.pem root-ed25519-priv.der root-ed25519-priv.pem root-ed25519.der root-ed25519.pem server-ed25519-cert.pem server-ed25519-key.der server-ed25519-key.pem server-ed25519-priv.der server-ed25519-priv.pem server-ed25519.der server-ed25519.pem
ed448 ca-ed448-key.der ca-ed448-key.pem ca-ed448-priv.der ca-ed448-priv.pem ca-ed448.der ca-ed448.pem client-ed448-key.der client-ed448-key.pem client-ed448-priv.der client-ed448-priv.pem client-ed448.der client-ed448.pem gen-ed448-certs.sh gen-ed448-keys.sh include.am root-ed448-key.der root-ed448-key.pem root-ed448-priv.der root-ed448-priv.pem root-ed448.der root-ed448.pem server-ed448-cert.pem server-ed448-key.der server-ed448-key.pem server-ed448-priv.der server-ed448-priv.pem server-ed448.der server-ed448.pem
external DigiCertGlobalRootCA.pem README.txt ca-digicert-ev.pem ca-globalsign-root.pem ca-google-root.pem ca_collection.pem include.am
falcon bench_falcon_level1_key.der bench_falcon_level5_key.der include.am
intermediate
ca_false_intermediate gentestcert.sh int_ca.key server.key test_ca.key test_ca.pem test_int_not_cacert.pem test_sign_bynoca_srv.pem wolfssl_base.conf wolfssl_srv.conf
ca-ecc-bad-aki.der ca-ecc-bad-aki.pem ca-int-cert.der ca-int-cert.pem ca-int-ecc-cert.der ca-int-ecc-cert.pem ca-int-ecc-key.der ca-int-ecc-key.pem ca-int-key.der ca-int-key.pem ca-int2-cert.der ca-int2-cert.pem ca-int2-ecc-cert.der ca-int2-ecc-cert.pem ca-int2-ecc-key.der ca-int2-ecc-key.pem ca-int2-key.der ca-int2-key.pem client-chain-alt-ecc.pem client-chain-alt.pem client-chain-ecc.der client-chain-ecc.pem client-chain.der client-chain.pem client-int-cert.der client-int-cert.pem client-int-ecc-cert.der client-int-ecc-cert.pem genintcerts.sh include.am server-chain-alt-ecc.pem server-chain-alt.pem server-chain-ecc.der server-chain-ecc.pem server-chain-short.pem server-chain.der server-chain.pem server-int-cert.der server-int-cert.pem server-int-ecc-cert.der server-int-ecc-cert.pem
lms bc_hss_L2_H5_W8_root.der bc_hss_L3_H5_W4_root.der bc_lms_chain_ca.der bc_lms_chain_leaf.der bc_lms_native_bc_root.der bc_lms_sha256_h10_w8_root.der bc_lms_sha256_h5_w4_root.der include.am
mldsa README.txt include.am mldsa44-cert.der mldsa44-cert.pem mldsa44-key.pem mldsa44_bare-priv.der mldsa44_bare-seed.der mldsa44_oqskeypair.der mldsa44_priv-only.der mldsa44_pub-spki.der mldsa44_seed-only.der mldsa44_seed-priv.der mldsa65-cert.der mldsa65-cert.pem mldsa65-key.pem mldsa65_bare-priv.der mldsa65_bare-seed.der mldsa65_oqskeypair.der mldsa65_priv-only.der mldsa65_pub-spki.der mldsa65_seed-only.der mldsa65_seed-priv.der mldsa87-cert.der mldsa87-cert.pem mldsa87-key.pem mldsa87_bare-priv.der mldsa87_bare-seed.der mldsa87_oqskeypair.der mldsa87_priv-only.der mldsa87_pub-spki.der mldsa87_seed-only.der mldsa87_seed-priv.der
ocsp imposter-root-ca-cert.der imposter-root-ca-cert.pem imposter-root-ca-key.der imposter-root-ca-key.pem include.am index-ca-and-intermediate-cas.txt index-ca-and-intermediate-cas.txt.attr index-intermediate1-ca-issued-certs.txt index-intermediate1-ca-issued-certs.txt.attr index-intermediate2-ca-issued-certs.txt index-intermediate2-ca-issued-certs.txt.attr index-intermediate3-ca-issued-certs.txt index-intermediate3-ca-issued-certs.txt.attr intermediate1-ca-cert.der intermediate1-ca-cert.pem intermediate1-ca-key.der intermediate1-ca-key.pem intermediate2-ca-cert.der intermediate2-ca-cert.pem intermediate2-ca-key.der intermediate2-ca-key.pem intermediate3-ca-cert.der intermediate3-ca-cert.pem intermediate3-ca-key.der intermediate3-ca-key.pem ocsp-responder-cert.der ocsp-responder-cert.pem ocsp-responder-key.der ocsp-responder-key.pem openssl.cnf renewcerts-for-test.sh renewcerts.sh root-ca-cert.der root-ca-cert.pem root-ca-crl.pem root-ca-key.der root-ca-key.pem server1-cert.der server1-cert.pem server1-chain-noroot.pem server1-key.der server1-key.pem server2-cert.der server2-cert.pem server2-key.der server2-key.pem server3-cert.der server3-cert.pem server3-key.der server3-key.pem server4-cert.der server4-cert.pem server4-key.der server4-key.pem server5-cert.der server5-cert.pem server5-key.der server5-key.pem test-leaf-response.der test-multi-response.der test-response-nointern.der test-response-rsapss.der test-response.der
p521 ca-p521-key.der ca-p521-key.pem ca-p521-priv.der ca-p521-priv.pem ca-p521.der ca-p521.pem client-p521-key.der client-p521-key.pem client-p521-priv.der client-p521-priv.pem client-p521.der client-p521.pem gen-p521-certs.sh gen-p521-keys.sh include.am root-p521-key.der root-p521-key.pem root-p521-priv.der root-p521-priv.pem root-p521.der root-p521.pem server-p521-cert.pem server-p521-key.der server-p521-key.pem server-p521-priv.der server-p521-priv.pem server-p521.der server-p521.pem
renewcerts wolfssl.cnf
rpk client-cert-rpk.der client-ecc-cert-rpk.der include.am server-cert-rpk.der server-ecc-cert-rpk.der
rsapss ca-3072-rsapss-key.der ca-3072-rsapss-key.pem ca-3072-rsapss-priv.der ca-3072-rsapss-priv.pem ca-3072-rsapss.der ca-3072-rsapss.pem ca-rsapss-key.der ca-rsapss-key.pem ca-rsapss-priv.der ca-rsapss-priv.pem ca-rsapss.der ca-rsapss.pem client-3072-rsapss-key.der client-3072-rsapss-key.pem client-3072-rsapss-priv.der client-3072-rsapss-priv.pem client-3072-rsapss.der client-3072-rsapss.pem client-rsapss-key.der client-rsapss-key.pem client-rsapss-priv.der client-rsapss-priv.pem client-rsapss.der client-rsapss.pem gen-rsapss-keys.sh include.am renew-rsapss-certs.sh root-3072-rsapss-key.der root-3072-rsapss-key.pem root-3072-rsapss-priv.der root-3072-rsapss-priv.pem root-3072-rsapss.der root-3072-rsapss.pem root-rsapss-key.der root-rsapss-key.pem root-rsapss-priv.der root-rsapss-priv.pem root-rsapss.der root-rsapss.pem server-3072-rsapss-cert.pem server-3072-rsapss-key.der server-3072-rsapss-key.pem server-3072-rsapss-priv.der server-3072-rsapss-priv.pem server-3072-rsapss.der server-3072-rsapss.pem server-mix-rsapss-cert.pem server-rsapss-cert.pem server-rsapss-key.der server-rsapss-key.pem server-rsapss-priv.der server-rsapss-priv.pem server-rsapss.der server-rsapss.pem
sia timestamping-sia-cert.pem
slhdsa bench_slhdsa_sha2_128f_key.der bench_slhdsa_sha2_128s_key.der bench_slhdsa_sha2_192f_key.der bench_slhdsa_sha2_192s_key.der bench_slhdsa_sha2_256f_key.der bench_slhdsa_sha2_256s_key.der bench_slhdsa_shake128f_key.der bench_slhdsa_shake128s_key.der bench_slhdsa_shake192f_key.der bench_slhdsa_shake192s_key.der bench_slhdsa_shake256f_key.der bench_slhdsa_shake256s_key.der client-mldsa44-priv.pem client-mldsa44-sha2.der client-mldsa44-sha2.pem client-mldsa44-shake.der client-mldsa44-shake.pem gen-slhdsa-mldsa-certs.sh include.am root-slhdsa-sha2-128s-priv.der root-slhdsa-sha2-128s-priv.pem root-slhdsa-sha2-128s.der root-slhdsa-sha2-128s.pem root-slhdsa-shake-128s-priv.der root-slhdsa-shake-128s-priv.pem root-slhdsa-shake-128s.der root-slhdsa-shake-128s.pem server-mldsa44-priv.pem server-mldsa44-sha2.der server-mldsa44-sha2.pem server-mldsa44-shake.der server-mldsa44-shake.pem
sm2 ca-sm2-key.der ca-sm2-key.pem ca-sm2-priv.der ca-sm2-priv.pem ca-sm2.der ca-sm2.pem client-sm2-key.der client-sm2-key.pem client-sm2-priv.der client-sm2-priv.pem client-sm2.der client-sm2.pem fix_sm2_spki.py gen-sm2-certs.sh gen-sm2-keys.sh include.am root-sm2-key.der root-sm2-key.pem root-sm2-priv.der root-sm2-priv.pem root-sm2.der root-sm2.pem self-sm2-cert.pem self-sm2-key.pem self-sm2-priv.pem server-sm2-cert.der server-sm2-cert.pem server-sm2-key.der server-sm2-key.pem server-sm2-priv.der server-sm2-priv.pem server-sm2.der server-sm2.pem
statickeys dh-ffdhe2048-params.pem dh-ffdhe2048-pub.der dh-ffdhe2048-pub.pem dh-ffdhe2048.der dh-ffdhe2048.pem ecc-secp256r1.der ecc-secp256r1.pem gen-static.sh include.am x25519-pub.der x25519-pub.pem x25519.der x25519.pem
test
expired expired-ca.der expired-ca.pem expired-cert.der expired-cert.pem
catalog.txt cert-bad-neg-int.der cert-bad-oid.der cert-bad-utf8.der cert-ext-ia.cfg cert-ext-ia.der cert-ext-ia.pem cert-ext-joi.cfg cert-ext-joi.der cert-ext-joi.pem cert-ext-mnc.der cert-ext-multiple.cfg cert-ext-multiple.der cert-ext-multiple.pem cert-ext-nc-combined.der cert-ext-nc-combined.pem cert-ext-nc.cfg cert-ext-nc.der cert-ext-nc.pem cert-ext-ncdns.der cert-ext-ncdns.pem cert-ext-ncip.der cert-ext-ncip.pem cert-ext-ncmixed.der cert-ext-ncmulti.der cert-ext-ncmulti.pem cert-ext-ncrid.der cert-ext-ncrid.pem cert-ext-nct.cfg cert-ext-nct.der cert-ext-nct.pem cert-ext-ndir-exc.cfg cert-ext-ndir-exc.der cert-ext-ndir-exc.pem cert-ext-ndir.cfg cert-ext-ndir.der cert-ext-ndir.pem cert-ext-ns.der cert-over-max-altnames.cfg cert-over-max-altnames.der cert-over-max-altnames.pem cert-over-max-nc.cfg cert-over-max-nc.der cert-over-max-nc.pem client-ecc-cert-ski.hex cn-ip-literal.der cn-ip-wildcard.der crit-cert.pem crit-key.pem dh1024.der dh1024.pem dh512.der dh512.pem digsigku.pem encrypteddata.msg gen-badsig.sh gen-ext-certs.sh gen-testcerts.sh include.am kari-keyid-cms.msg ktri-keyid-cms.msg ossl-trusted-cert.pem server-badaltname.der server-badaltname.pem server-badaltnull.der server-badaltnull.pem server-badcn.der server-badcn.pem server-badcnnull.der server-badcnnull.pem server-cert-ecc-badsig.der server-cert-ecc-badsig.pem server-cert-rsa-badsig.der server-cert-rsa-badsig.pem server-duplicate-policy.pem server-garbage.der server-garbage.pem server-goodalt.der server-goodalt.pem server-goodaltwild.der server-goodaltwild.pem server-goodcn.der server-goodcn.pem server-goodcnwild.der server-goodcnwild.pem server-localhost.der server-localhost.pem smime-test-canon.p7s smime-test-multipart-badsig.p7s smime-test-multipart.p7s smime-test.p7s
test-pathlen assemble-chains.sh chainA-ICA1-key.pem chainA-ICA1-pathlen0.pem chainA-assembled.pem chainA-entity-key.pem chainA-entity.pem chainB-ICA1-key.pem chainB-ICA1-pathlen0.pem chainB-ICA2-key.pem chainB-ICA2-pathlen1.pem chainB-assembled.pem chainB-entity-key.pem chainB-entity.pem chainC-ICA1-key.pem chainC-ICA1-pathlen1.pem chainC-assembled.pem chainC-entity-key.pem chainC-entity.pem chainD-ICA1-key.pem chainD-ICA1-pathlen127.pem chainD-assembled.pem chainD-entity-key.pem chainD-entity.pem chainE-ICA1-key.pem chainE-ICA1-pathlen128.pem chainE-assembled.pem chainE-entity-key.pem chainE-entity.pem chainF-ICA1-key.pem chainF-ICA1-pathlen1.pem chainF-ICA2-key.pem chainF-ICA2-pathlen0.pem chainF-assembled.pem chainF-entity-key.pem chainF-entity.pem chainG-ICA1-key.pem chainG-ICA1-pathlen0.pem chainG-ICA2-key.pem chainG-ICA2-pathlen1.pem chainG-ICA3-key.pem chainG-ICA3-pathlen99.pem chainG-ICA4-key.pem chainG-ICA4-pathlen5.pem chainG-ICA5-key.pem chainG-ICA5-pathlen20.pem chainG-ICA6-key.pem chainG-ICA6-pathlen10.pem chainG-ICA7-key.pem chainG-ICA7-pathlen100.pem chainG-assembled.pem chainG-entity-key.pem chainG-entity.pem chainH-ICA1-key.pem chainH-ICA1-pathlen0.pem chainH-ICA2-key.pem chainH-ICA2-pathlen2.pem chainH-ICA3-key.pem chainH-ICA3-pathlen2.pem chainH-ICA4-key.pem chainH-ICA4-pathlen2.pem chainH-assembled.pem chainH-entity-key.pem chainH-entity.pem chainI-ICA1-key.pem chainI-ICA1-no_pathlen.pem chainI-ICA2-key.pem chainI-ICA2-no_pathlen.pem chainI-ICA3-key.pem chainI-ICA3-pathlen2.pem chainI-assembled.pem chainI-entity-key.pem chainI-entity.pem chainJ-ICA1-key.pem chainJ-ICA1-no_pathlen.pem chainJ-ICA2-key.pem chainJ-ICA2-no_pathlen.pem chainJ-ICA3-key.pem chainJ-ICA3-no_pathlen.pem chainJ-ICA4-key.pem chainJ-ICA4-pathlen2.pem chainJ-assembled.pem chainJ-entity-key.pem chainJ-entity.pem include.am refreshkeys.sh
test-serial0 ee_normal.pem ee_serial0.pem generate_certs.sh include.am intermediate_serial0.pem root_serial0.pem root_serial0_key.pem selfsigned_nonca_serial0.pem
xmss bc_xmss_chain_ca.der bc_xmss_chain_leaf.der bc_xmss_sha2_10_256_root.der bc_xmss_sha2_16_256_root.der bc_xmssmt_sha2_20_2_256_root.der bc_xmssmt_sha2_20_4_256_root.der bc_xmssmt_sha2_40_8_256_root.der include.am
ca-cert-chain.der ca-cert.der ca-cert.pem ca-ecc-cert.der ca-ecc-cert.pem ca-ecc-key.der ca-ecc-key.pem ca-ecc384-cert.der ca-ecc384-cert.pem ca-ecc384-key.der ca-ecc384-key.pem ca-key-pkcs8-attribute.der ca-key.der ca-key.pem check_dates.sh client-absolute-urn.pem client-ca-cert.der client-ca-cert.pem client-ca.pem client-cert-ext.der client-cert-ext.pem client-cert.der client-cert.pem client-crl-dist.der client-crl-dist.pem client-ecc-ca-cert.der client-ecc-ca-cert.pem client-ecc-cert.der client-ecc-cert.pem client-ecc384-cert.der client-ecc384-cert.pem client-ecc384-key.der client-ecc384-key.pem client-key.der client-key.pem client-keyEnc.pem client-keyPub.der client-keyPub.pem client-relative-uri.pem client-uri-cert.pem csr.attr.der csr.dsa.der csr.dsa.pem csr.ext.der csr.signed.der dh-priv-2048.der dh-priv-2048.pem dh-pub-2048.der dh-pub-2048.pem dh-pubkey-2048.der dh2048.der dh2048.pem dh3072.der dh3072.pem dh4096.der dh4096.pem dsa-pubkey-2048.der dsa2048.der dsa2048.pem dsa3072.der dsaparams.der dsaparams.pem ecc-client-key.der ecc-client-key.pem ecc-client-keyPub.der ecc-client-keyPub.pem ecc-key-comp.pem ecc-keyPkcs8.der ecc-keyPkcs8.pem ecc-keyPkcs8Enc.der ecc-keyPkcs8Enc.pem ecc-keyPub.der ecc-keyPub.pem ecc-params.der ecc-params.pem ecc-privOnlyCert.pem ecc-privOnlyKey.pem ecc-privkey.der ecc-privkey.pem ecc-privkeyPkcs8.der ecc-privkeyPkcs8.pem ecc-rsa-server.p12 empty-issuer-cert.pem entity-no-ca-bool-cert.pem entity-no-ca-bool-key.pem fpki-cert.der fpki-certpol-cert.der gen_revoked.sh include.am renewcerts.sh rid-cert.der rsa-pub-2048.pem rsa2048.der rsa3072.der server-cert-chain.der server-cert.der server-cert.pem server-ecc-comp.der server-ecc-comp.pem server-ecc-rsa.der server-ecc-rsa.pem server-ecc-self.der server-ecc-self.pem server-ecc.der server-ecc.pem server-ecc384-cert.der server-ecc384-cert.pem server-ecc384-key.der server-ecc384-key.pem server-key.der server-key.pem server-keyEnc.pem server-keyPkcs8.der server-keyPkcs8.pem server-keyPkcs8Enc.der server-keyPkcs8Enc.pem server-keyPkcs8Enc12.pem server-keyPkcs8Enc2.pem server-keyPub.der server-keyPub.pem server-revoked-cert.pem server-revoked-key.pem taoCert.txt test-ber-exp02-05-2022.p7b test-degenerate.p7b test-multiple-recipients.p7b test-servercert-rc2.p12 test-servercert.p12 test-stream-dec.p7b test-stream-sign.p7b wolfssl-website-ca.pem x942dh2048.der x942dh2048.pem
cmake
consumer CMakeLists.txt README.md main.c
modules FindARIA.cmake FindOQS.cmake
Config.cmake.in README.md config.in functions.cmake include.am options.h.in wolfssl-config-version.cmake.in wolfssl-targets.cmake.in
debian
source format
changelog.in control.in copyright include.am libwolfssl-dev.install libwolfssl.install rules.in
doc
dox_comments
header_files aes.h arc4.h ascon.h asn.h asn_public.h blake2.h bn.h camellia.h chacha.h chacha20_poly1305.h cmac.h coding.h compress.h cryptocb.h curve25519.h curve448.h des3.h dh.h doxygen_groups.h doxygen_pages.h dsa.h ecc.h eccsi.h ed25519.h ed448.h error-crypt.h evp.h hash.h hmac.h iotsafe.h kdf.h logging.h md2.h md4.h md5.h memory.h ocsp.h pem.h pkcs11.h pkcs7.h poly1305.h psa.h puf.h pwdbased.h quic.h random.h ripemd.h rsa.h sakke.h sha.h sha256.h sha3.h sha512.h signature.h siphash.h srp.h ssl.h tfm.h types.h wc_encrypt.h wc_port.h wc_she.h wc_slhdsa.h wolfio.h
header_files-ja aes.h arc4.h ascon.h asn.h asn_public.h blake2.h bn.h camellia.h chacha.h chacha20_poly1305.h cmac.h coding.h compress.h cryptocb.h curve25519.h curve448.h des3.h dh.h doxygen_groups.h doxygen_pages.h dsa.h ecc.h eccsi.h ed25519.h ed448.h error-crypt.h evp.h hash.h hmac.h iotsafe.h kdf.h logging.h md2.h md4.h md5.h memory.h ocsp.h pem.h pkcs11.h pkcs7.h poly1305.h psa.h pwdbased.h quic.h random.h ripemd.h rsa.h sakke.h sha.h sha256.h sha3.h sha512.h signature.h siphash.h srp.h ssl.h tfm.h types.h wc_encrypt.h wc_port.h wolfio.h
formats
html
html_changes
search search.css search.js
customdoxygen.css doxygen.css menu.js menudata.js tabs.css
Doxyfile footer.html header.html mainpage.dox
pdf Doxyfile header.tex
images wolfssl_logo.png
QUIC.md README.txt README_DOXYGEN check_api.sh generate_documentation.sh include.am
examples
asn1 asn1.c dumpasn1.cfg gen_oid_names.rb include.am oid_names.h
async Makefile README.md async_client.c async_server.c async_tls.c async_tls.h include.am user_settings.h
benchmark include.am tls_bench.c tls_bench.h
client client.c client.h client.sln client.vcproj client.vcxproj include.am
configs README.md include.am user_settings_EBSnet.h user_settings_all.h user_settings_arduino.h user_settings_baremetal.h user_settings_ca.h user_settings_curve25519nonblock.h user_settings_dtls13.h user_settings_eccnonblock.h user_settings_espressif.h user_settings_fipsv2.h user_settings_fipsv5.h user_settings_min_ecc.h user_settings_openssl_compat.h user_settings_pkcs7.h user_settings_platformio.h user_settings_pq.h user_settings_rsa_only.h user_settings_stm32.h user_settings_template.h user_settings_tls12.h user_settings_tls13.h user_settings_wolfboot_keytools.h user_settings_wolfssh.h user_settings_wolftpm.h
crypto_policies
default wolfssl.txt
future wolfssl.txt
legacy wolfssl.txt
echoclient echoclient.c echoclient.h echoclient.sln echoclient.vcproj echoclient.vcxproj include.am quit
echoserver echoserver.c echoserver.h echoserver.sln echoserver.vcproj echoserver.vcxproj include.am
ocsp_responder include.am ocsp_responder.c ocsp_responder.h
pem include.am pem.c
sctp include.am sctp-client-dtls.c sctp-client.c sctp-server-dtls.c sctp-server.c
server include.am server.c server.h server.sln server.vcproj server.vcxproj
README.md include.am
linuxkm
patches
5.10.17 WOLFSSL_LINUXKM_HAVE_GET_RANDOM_CALLBACKS-5v10v17.patch
5.10.236 WOLFSSL_LINUXKM_HAVE_GET_RANDOM_CALLBACKS-5v10v236.patch
5.14.0-570.58.1.el9_6 WOLFSSL_LINUXKM_HAVE_GET_RANDOM_CALLBACKS-5v14-570v58v1-el9_6.patch
5.15 WOLFSSL_LINUXKM_HAVE_GET_RANDOM_CALLBACKS-5v15.patch
5.17 WOLFSSL_LINUXKM_HAVE_GET_RANDOM_CALLBACKS-5v17.patch
5.17-ubuntu-jammy-tegra WOLFSSL_LINUXKM_HAVE_GET_RANDOM_CALLBACKS-5v17-ubuntu-jammy-tegra.patch
6.1.73 WOLFSSL_LINUXKM_HAVE_GET_RANDOM_CALLBACKS-6v1v73.patch
6.12 WOLFSSL_LINUXKM_HAVE_GET_RANDOM_CALLBACKS-6v12.patch
6.15 WOLFSSL_LINUXKM_HAVE_GET_RANDOM_CALLBACKS-6v15.patch
7.0 WOLFSSL_LINUXKM_HAVE_GET_RANDOM_CALLBACKS-7v0.patch
regen-patches.sh
Kbuild Makefile README.md get_thread_size.c include.am linuxkm-fips-hash-wrapper.sh linuxkm-fips-hash.c linuxkm_memory.c linuxkm_memory.h linuxkm_wc_port.h lkcapi_aes_glue.c lkcapi_dh_glue.c lkcapi_ecdh_glue.c lkcapi_ecdsa_glue.c lkcapi_glue.c lkcapi_rsa_glue.c lkcapi_sha_glue.c module_exports.c.template module_hooks.c pie_redirect_table.c wolfcrypt.lds x86_vector_register_glue.c
m4 ax_add_am_macro.m4 ax_am_jobserver.m4 ax_am_macros.m4 ax_append_compile_flags.m4 ax_append_flag.m4 ax_append_link_flags.m4 ax_append_to_file.m4 ax_atomic.m4 ax_bsdkm.m4 ax_check_compile_flag.m4 ax_check_link_flag.m4 ax_compiler_version.m4 ax_count_cpus.m4 ax_create_generic_config.m4 ax_debug.m4 ax_file_escapes.m4 ax_harden_compiler_flags.m4 ax_linuxkm.m4 ax_print_to_file.m4 ax_pthread.m4 ax_require_defined.m4 ax_tls.m4 ax_vcs_checkout.m4 hexversion.m4 lib_socket_nsl.m4 visibility.m4
mcapi
wolfcrypt_mcapi.X
nbproject configurations.xml include.am project.xml
Makefile
wolfcrypt_test.X
nbproject configurations.xml include.am project.xml
Makefile
wolfssl.X
nbproject configurations.xml include.am project.xml
Makefile
zlib.X
nbproject configurations.xml include.am project.xml
Makefile
PIC32MZ-serial.h README crypto.c crypto.h include.am mcapi_test.c user_settings.h
mplabx
wolfcrypt_benchmark.X
nbproject configurations.xml include.am project.xml
Makefile
wolfcrypt_test.X
nbproject configurations.xml include.am project.xml
Makefile
wolfssl.X
nbproject configurations.xml include.am project.xml
Makefile
PIC32MZ-serial.h README benchmark_main.c include.am test_main.c user_settings.h
mqx
util_lib
Sources include.am util.c util.h
wolfcrypt_benchmark
Debugger K70FN1M0.mem init_kinetis.tcl mass_erase_kinetis.tcl
Sources include.am main.c main.h
ReferencedRSESystems.xml wolfcrypt_benchmark_twrk70f120m_Int_Flash_DDRData_Debug_PnE_U-MultiLink.launch wolfcrypt_benchmark_twrk70f120m_Int_Flash_DDRData_Release_PnE_U-MultiLink.launch wolfcrypt_benchmark_twrk70f120m_Int_Flash_SramData_Debug_JTrace.jlink wolfcrypt_benchmark_twrk70f120m_Int_Flash_SramData_Debug_JTrace.launch wolfcrypt_benchmark_twrk70f120m_Int_Flash_SramData_Debug_PnE_U-MultiLink.launch wolfcrypt_benchmark_twrk70f120m_Int_Flash_SramData_Release_PnE_U-MultiLink.launch
wolfcrypt_test
Debugger K70FN1M0.mem init_kinetis.tcl mass_erase_kinetis.tcl
Sources include.am main.c main.h
ReferencedRSESystems.xml wolfcrypt_test_twrk70f120m_Int_Flash_DDRData_Debug_PnE_U-MultiLink.launch wolfcrypt_test_twrk70f120m_Int_Flash_DDRData_Release_PnE_U-MultiLink.launch wolfcrypt_test_twrk70f120m_Int_Flash_SramData_Debug_JTrace.jlink wolfcrypt_test_twrk70f120m_Int_Flash_SramData_Debug_JTrace.launch wolfcrypt_test_twrk70f120m_Int_Flash_SramData_Debug_PnE_U-MultiLink.launch wolfcrypt_test_twrk70f120m_Int_Flash_SramData_Release_PnE_U-MultiLink.launch
wolfssl include.am
wolfssl_client
Debugger K70FN1M0.mem init_kinetis.tcl mass_erase_kinetis.tcl
Sources include.am main.c main.h
ReferencedRSESystems.xml wolfssl_client_twrk70f120m_Int_Flash_DDRData_Debug_PnE_U-MultiLink.launch wolfssl_client_twrk70f120m_Int_Flash_DDRData_Release_PnE_U-MultiLink.launch wolfssl_client_twrk70f120m_Int_Flash_SramData_Debug_JTrace.jlink wolfssl_client_twrk70f120m_Int_Flash_SramData_Debug_JTrace.launch wolfssl_client_twrk70f120m_Int_Flash_SramData_Debug_PnE_U-MultiLink.launch wolfssl_client_twrk70f120m_Int_Flash_SramData_Release_PnE_U-MultiLink.launch
README
rpm include.am spec.in
scripts
bench bench_functions.sh
aria-cmake-build-test.sh asn1_oid_sum.pl benchmark.test benchmark_compare.sh cleanup_testfiles.sh crl-gen-openssl.test crl-revoked.test dertoc.pl dtls.test dtlscid.test external.test google.test include.am makedistsmall.sh memtest.sh ocsp-responder-openssl-interop.test ocsp-stapling-with-ca-as-responder.test ocsp-stapling-with-wolfssl-responder.test ocsp-stapling.test ocsp-stapling2.test ocsp-stapling_tls13multi.test ocsp.test openssl.test openssl_srtp.test pem.test ping.test pkcallbacks.test psk.test resume.test rsapss.test sniffer-gen.sh sniffer-ipv6.pcap sniffer-static-rsa.pcap sniffer-testsuite.test sniffer-tls12-keylog.out sniffer-tls12-keylog.pcap sniffer-tls12-keylog.sslkeylog sniffer-tls13-dh-resume.pcap sniffer-tls13-dh.pcap sniffer-tls13-ecc-resume.pcap sniffer-tls13-ecc.pcap sniffer-tls13-hrr.pcap sniffer-tls13-keylog.out sniffer-tls13-keylog.pcap sniffer-tls13-keylog.sslkeylog sniffer-tls13-x25519-resume.pcap sniffer-tls13-x25519.pcap stm32l4-v4_0_1_build.sh tls13.test trusted_peer.test unit.test.in user_settings_asm.sh
src bio.c conf.c crl.c dtls.c dtls13.c include.am internal.c keys.c ocsp.c pk.c pk_ec.c pk_rsa.c quic.c sniffer.c ssl.c ssl_api_cert.c ssl_api_crl_ocsp.c ssl_api_pk.c ssl_asn1.c ssl_bn.c ssl_certman.c ssl_crypto.c ssl_ech.c ssl_load.c ssl_misc.c ssl_p7p12.c ssl_sess.c ssl_sk.c tls.c tls13.c wolfio.c x509.c x509_str.c
sslSniffer
sslSnifferTest README_WIN.md include.am snifftest.c sslSniffTest.vcproj sslSniffTest.vcxproj
README.md sslSniffer.vcproj sslSniffer.vcxproj
support gen-debug-trace-error-codes.sh include.am wolfssl.pc.in
tests
api api.h api_decl.h create_ocsp_test_blobs.py include.am test_aes.c test_aes.h test_arc4.c test_arc4.h test_ascon.c test_ascon.h test_ascon_kats.h test_asn.c test_asn.h test_blake2.c test_blake2.h test_camellia.c test_camellia.h test_certman.c test_certman.h test_chacha.c test_chacha.h test_chacha20_poly1305.c test_chacha20_poly1305.h test_cmac.c test_cmac.h test_curve25519.c test_curve25519.h test_curve448.c test_curve448.h test_des3.c test_des3.h test_dh.c test_dh.h test_digest.h test_dsa.c test_dsa.h test_dtls.c test_dtls.h test_ecc.c test_ecc.h test_ed25519.c test_ed25519.h test_ed448.c test_ed448.h test_evp.c test_evp.h test_evp_cipher.c test_evp_cipher.h test_evp_digest.c test_evp_digest.h test_evp_pkey.c test_evp_pkey.h test_hash.c test_hash.h test_hmac.c test_hmac.h test_md2.c test_md2.h test_md4.c test_md4.h test_md5.c test_md5.h test_mldsa.c test_mldsa.h test_mlkem.c test_mlkem.h test_ocsp.c test_ocsp.h test_ocsp_test_blobs.h test_ossl_asn1.c test_ossl_asn1.h test_ossl_bio.c test_ossl_bio.h test_ossl_bn.c test_ossl_bn.h test_ossl_cipher.c test_ossl_cipher.h test_ossl_dgst.c test_ossl_dgst.h test_ossl_dh.c test_ossl_dh.h test_ossl_dsa.c test_ossl_dsa.h test_ossl_ec.c test_ossl_ec.h test_ossl_ecx.c test_ossl_ecx.h test_ossl_mac.c test_ossl_mac.h test_ossl_obj.c test_ossl_obj.h test_ossl_p7p12.c test_ossl_p7p12.h test_ossl_pem.c test_ossl_pem.h test_ossl_rand.c test_ossl_rand.h test_ossl_rsa.c test_ossl_rsa.h test_ossl_sk.c test_ossl_sk.h test_ossl_x509.c test_ossl_x509.h test_ossl_x509_acert.c test_ossl_x509_acert.h test_ossl_x509_crypto.c test_ossl_x509_crypto.h test_ossl_x509_ext.c test_ossl_x509_ext.h test_ossl_x509_info.c test_ossl_x509_info.h test_ossl_x509_io.c test_ossl_x509_io.h test_ossl_x509_lu.c test_ossl_x509_lu.h test_ossl_x509_name.c test_ossl_x509_name.h test_ossl_x509_pk.c test_ossl_x509_pk.h test_ossl_x509_str.c test_ossl_x509_str.h test_ossl_x509_vp.c test_ossl_x509_vp.h test_pkcs12.c test_pkcs12.h test_pkcs7.c test_pkcs7.h test_poly1305.c test_poly1305.h test_random.c test_random.h test_rc2.c test_rc2.h test_ripemd.c test_ripemd.h test_rsa.c test_rsa.h test_sha.c test_sha.h test_sha256.c test_sha256.h test_sha3.c test_sha3.h test_sha512.c test_sha512.h test_she.c test_she.h test_signature.c test_signature.h test_slhdsa.c test_slhdsa.h test_sm2.c test_sm2.h test_sm3.c test_sm3.h test_sm4.c test_sm4.h test_tls.c test_tls.h test_tls13.c test_tls13.h test_tls_ext.c test_tls_ext.h test_wc_encrypt.c test_wc_encrypt.h test_wolfmath.c test_wolfmath.h test_x509.c test_x509.h
emnet
IP IP.h
Makefile emnet_nonblock_test.c emnet_shim.c
freertos-mem-track-repro FreeRTOS.h repro.c run.sh semphr.h task.h user_settings.h
swdev .gitignore Makefile README.md swdev.c swdev.h swdev_loader.c swdev_loader.h user_settings.h
CONF_FILES_README.md NCONF_test.cnf README TXT_DB.txt api.c include.am quic.c srp.c suites.c test-altchains.conf test-chains.conf test-dhprime.conf test-dtls-downgrade.conf test-dtls-fails-cipher.conf test-dtls-fails.conf test-dtls-group.conf test-dtls-mtu.conf test-dtls-reneg-client.conf test-dtls-reneg-server.conf test-dtls-resume.conf test-dtls-sha2.conf test-dtls-srtp-fails.conf test-dtls-srtp.conf test-dtls.conf test-dtls13-cid.conf test-dtls13-downgrade-fails.conf test-dtls13-downgrade.conf test-dtls13-pq-hybrid-extra-frag.conf test-dtls13-pq-hybrid-extra.conf test-dtls13-pq-hybrid-frag.conf test-dtls13-pq-standalone-frag.conf test-dtls13-pq-standalone.conf test-dtls13-psk.conf test-dtls13.conf test-ecc-cust-curves.conf test-ed25519.conf test-ed448.conf test-enckeys.conf test-fails.conf test-maxfrag-dtls.conf test-maxfrag.conf test-p521.conf test-psk-no-id-sha2.conf test-psk-no-id.conf test-psk.conf test-rsapss.conf test-sctp-sha2.conf test-sctp.conf test-sha2.conf test-sig.conf test-sm2.conf test-tls-downgrade.conf test-tls13-down.conf test-tls13-ecc.conf test-tls13-pq-hybrid-extra.conf test-tls13-pq-hybrid.conf test-tls13-pq-standalone.conf test-tls13-psk-certs.conf test-tls13-psk.conf test-tls13-slhdsa-fail.conf test-tls13-slhdsa-sha2.conf test-tls13-slhdsa-shake.conf test-tls13.conf test-trustpeer.conf test.conf unit.c unit.h utils.c utils.h w64wrapper.c
testsuite include.am testsuite.c testsuite.sln testsuite.vcproj testsuite.vcxproj utils.c utils.h
tirtos
packages
ti
net
wolfssl
tests
EK_TM4C1294XL
wolfcrypt
benchmark TM4C1294NC.icf benchmark.cfg main.c package.bld.hide package.xdc
test TM4C1294NC.icf main.c package.bld.hide package.xdc test.cfg
package.bld package.xdc package.xs
.gitignore README include.am products.mak wolfssl.bld wolfssl.mak
wolfcrypt
benchmark README.md benchmark-VS2022.sln benchmark-VS2022.vcxproj benchmark-VS2022.vcxproj.user benchmark.c benchmark.h benchmark.sln benchmark.vcproj benchmark.vcxproj include.am
src
port
Espressif
esp_crt_bundle README.md cacrt_all.pem cacrt_deprecated.pem cacrt_local.pem esp_crt_bundle.c gen_crt_bundle.py pio_install_cryptography.py
README.md esp32_aes.c esp32_mp.c esp32_sha.c esp32_util.c esp_sdk_mem_lib.c esp_sdk_time_lib.c esp_sdk_wifi_lib.c
Renesas README.md renesas_common.c renesas_fspsm_aes.c renesas_fspsm_rsa.c renesas_fspsm_sha.c renesas_fspsm_util.c renesas_rx64_hw_sha.c renesas_rx64_hw_util.c renesas_tsip_aes.c renesas_tsip_rsa.c renesas_tsip_sha.c renesas_tsip_util.c
af_alg afalg_aes.c afalg_hash.c wc_afalg.c
aria aria-crypt.c aria-cryptocb.c
arm armv8-32-aes-asm.S armv8-32-aes-asm_c.c armv8-32-chacha-asm.S armv8-32-chacha-asm_c.c armv8-32-curve25519.S armv8-32-curve25519_c.c armv8-32-mlkem-asm.S armv8-32-mlkem-asm_c.c armv8-32-poly1305-asm.S armv8-32-poly1305-asm_c.c armv8-32-sha256-asm.S armv8-32-sha256-asm_c.c armv8-32-sha3-asm.S armv8-32-sha3-asm_c.c armv8-32-sha512-asm.S armv8-32-sha512-asm_c.c armv8-aes-asm.S armv8-aes-asm_c.c armv8-aes.c armv8-chacha-asm.S armv8-chacha-asm_c.c armv8-curve25519.S armv8-curve25519_c.c armv8-mlkem-asm.S armv8-mlkem-asm_c.c armv8-poly1305-asm.S armv8-poly1305-asm_c.c armv8-sha256-asm.S armv8-sha256-asm_c.c armv8-sha256.c armv8-sha3-asm.S armv8-sha3-asm_c.c armv8-sha512-asm.S armv8-sha512-asm_c.c armv8-sha512.c cryptoCell.c cryptoCellHash.c thumb2-aes-asm.S thumb2-aes-asm_c.c thumb2-chacha-asm.S thumb2-chacha-asm_c.c thumb2-curve25519.S thumb2-curve25519_c.c thumb2-mlkem-asm.S thumb2-mlkem-asm_c.c thumb2-poly1305-asm.S thumb2-poly1305-asm_c.c thumb2-sha256-asm.S thumb2-sha256-asm_c.c thumb2-sha3-asm.S thumb2-sha3-asm_c.c thumb2-sha512-asm.S thumb2-sha512-asm_c.c
atmel README.md atmel.c
autosar README.md cryif.c crypto.c csm.c include.am test.c
caam README.md caam_aes.c caam_doc.pdf caam_driver.c caam_error.c caam_integrity.c caam_qnx.c caam_sha.c wolfcaam_aes.c wolfcaam_cmac.c wolfcaam_ecdsa.c wolfcaam_fsl_nxp.c wolfcaam_hash.c wolfcaam_hmac.c wolfcaam_init.c wolfcaam_qnx.c wolfcaam_rsa.c wolfcaam_seco.c wolfcaam_x25519.c
cavium README.md README_Octeon.md cavium_nitrox.c cavium_octeon_sync.c
cuda README.md aes-cuda.cu
cypress README.md psoc6_crypto.c
devcrypto README.md devcrypto_aes.c devcrypto_ecdsa.c devcrypto_hash.c devcrypto_hmac.c devcrypto_rsa.c devcrypto_x25519.c wc_devcrypto.c
intel README.md quickassist.c quickassist_mem.c quickassist_sync.c
iotsafe iotsafe.c
kcapi README.md kcapi_aes.c kcapi_dh.c kcapi_ecc.c kcapi_hash.c kcapi_hmac.c kcapi_rsa.c
liboqs liboqs.c
maxim README.md max3266x.c maxq10xx.c
mynewt mynewt_port.c
nxp README.md README_SE050.md casper_port.c dcp_port.c hashcrypt_port.c ksdk_port.c se050_port.c
pic32 pic32mz-crypt.c
ppc32 ppc32-sha256-asm.S ppc32-sha256-asm_c.c ppc32-sha256-asm_cr.c
psa README.md psa.c psa_aes.c psa_hash.c psa_pkcbs.c
riscv riscv-64-aes.c riscv-64-chacha.c riscv-64-poly1305.c riscv-64-sha256.c riscv-64-sha3.c riscv-64-sha512.c
rpi_pico README.md pico.c
silabs README.md silabs_aes.c silabs_ecc.c silabs_hash.c silabs_random.c
st README.md STM32MP13.md STM32MP25.md stm32.c stsafe.c
ti ti-aes.c ti-ccm.c ti-des3.c ti-hash.c
tropicsquare README.md tropic01.c
xilinx xil-aesgcm.c xil-sha3.c xil-versal-glue.c xil-versal-trng.c
nrf51.c
ASN_TEMPLATE.md aes.c aes_asm.S aes_asm.asm aes_gcm_asm.S aes_gcm_asm.asm aes_gcm_x86_asm.S aes_xts_asm.S aes_xts_asm.asm arc4.c ascon.c asm.c asn.c asn_orig.c async.c blake2b.c blake2s.c camellia.c chacha.c chacha20_poly1305.c chacha_asm.S chacha_asm.asm cmac.c coding.c compress.c cpuid.c cryptocb.c curve25519.c curve448.c des3.c dh.c dilithium.c dsa.c ecc.c ecc_fp.c eccsi.c ed25519.c ed448.c error.c evp.c evp_pk.c falcon.c fe_448.c fe_low_mem.c fe_operations.c fe_x25519_128.h fe_x25519_asm.S fp_mont_small.i fp_mul_comba_12.i fp_mul_comba_17.i fp_mul_comba_20.i fp_mul_comba_24.i fp_mul_comba_28.i fp_mul_comba_3.i fp_mul_comba_32.i fp_mul_comba_4.i fp_mul_comba_48.i fp_mul_comba_6.i fp_mul_comba_64.i fp_mul_comba_7.i fp_mul_comba_8.i fp_mul_comba_9.i fp_mul_comba_small_set.i fp_sqr_comba_12.i fp_sqr_comba_17.i fp_sqr_comba_20.i fp_sqr_comba_24.i fp_sqr_comba_28.i fp_sqr_comba_3.i fp_sqr_comba_32.i fp_sqr_comba_4.i fp_sqr_comba_48.i fp_sqr_comba_6.i fp_sqr_comba_64.i fp_sqr_comba_7.i fp_sqr_comba_8.i fp_sqr_comba_9.i fp_sqr_comba_small_set.i ge_448.c ge_low_mem.c ge_operations.c hash.c hmac.c hpke.c include.am integer.c kdf.c logging.c md2.c md4.c md5.c memory.c misc.c pkcs12.c pkcs7.c poly1305.c poly1305_asm.S poly1305_asm.asm puf.c pwdbased.c random.c rc2.c ripemd.c rng_bank.c rsa.c sakke.c sha.c sha256.c sha256_asm.S sha3.c sha3_asm.S sha512.c sha512_asm.S signature.c siphash.c sm2.c sm3.c sm3_asm.S sm4.c sp_arm32.c sp_arm64.c sp_armthumb.c sp_c32.c sp_c64.c sp_cortexm.c sp_dsp32.c sp_int.c sp_sm2_arm32.c sp_sm2_arm64.c sp_sm2_armthumb.c sp_sm2_c32.c sp_sm2_c64.c sp_sm2_cortexm.c sp_sm2_x86_64.c sp_sm2_x86_64_asm.S sp_x86_64.c sp_x86_64_asm.S sp_x86_64_asm.asm srp.c tfm.c wc_dsp.c wc_encrypt.c wc_lms.c wc_lms_impl.c wc_mldsa_asm.S wc_mlkem.c wc_mlkem_asm.S wc_mlkem_poly.c wc_pkcs11.c wc_port.c wc_she.c wc_slhdsa.c wc_xmss.c wc_xmss_impl.c wolfentropy.c wolfevent.c wolfmath.c
test README.md include.am test-VS2022.sln test-VS2022.vcxproj test-VS2022.vcxproj.user test.c test.h test.sln test.vcproj test_paths.h.in
wolfssl
openssl aes.h asn1.h asn1t.h bio.h bn.h buffer.h camellia.h cmac.h cms.h compat_types.h conf.h crypto.h des.h dh.h dsa.h ec.h ec25519.h ec448.h ecdh.h ecdsa.h ed25519.h ed448.h engine.h err.h evp.h fips_rand.h hmac.h include.am kdf.h lhash.h md4.h md5.h modes.h obj_mac.h objects.h ocsp.h opensslconf.h opensslv.h ossl_typ.h pem.h pkcs12.h pkcs7.h rand.h rc4.h ripemd.h rsa.h safestack.h sha.h sha3.h srp.h ssl.h ssl23.h stack.h tls1.h txt_db.h ui.h x509.h x509_vfy.h x509v3.h
wolfcrypt
port
Espressif esp-sdk-lib.h esp32-crypt.h esp_crt_bundle.h
Renesas renesas-fspsm-crypt.h renesas-fspsm-types.h renesas-rx64-hw-crypt.h renesas-tsip-crypt.h renesas_cmn.h renesas_fspsm_internal.h renesas_sync.h renesas_tsip_internal.h renesas_tsip_types.h
af_alg afalg_hash.h wc_afalg.h
aria aria-crypt.h aria-cryptocb.h
arm cryptoCell.h
atmel atmel.h
autosar CryIf.h Crypto.h Csm.h StandardTypes.h
caam caam_driver.h caam_error.h caam_qnx.h wolfcaam.h wolfcaam_aes.h wolfcaam_cmac.h wolfcaam_ecdsa.h wolfcaam_fsl_nxp.h wolfcaam_hash.h wolfcaam_qnx.h wolfcaam_rsa.h wolfcaam_seco.h wolfcaam_sha.h wolfcaam_x25519.h
cavium cavium_nitrox.h cavium_octeon_sync.h
cypress psoc6_crypto.h
devcrypto wc_devcrypto.h
intel quickassist.h quickassist_mem.h quickassist_sync.h
iotsafe iotsafe.h
kcapi kcapi_dh.h kcapi_ecc.h kcapi_hash.h kcapi_hmac.h kcapi_rsa.h wc_kcapi.h
liboqs liboqs.h
maxim max3266x-cryptocb.h max3266x.h maxq10xx.h
nxp casper_port.h dcp_port.h hashcrypt_port.h ksdk_port.h se050_port.h
pic32 pic32mz-crypt.h
psa psa.h
riscv riscv-64-asm.h
rpi_pico pico.h
silabs silabs_aes.h silabs_ecc.h silabs_hash.h silabs_random.h
st stm32.h stsafe.h
ti ti-ccm.h ti-hash.h
tropicsquare tropic01.h
xilinx xil-sha3.h xil-versal-glue.h xil-versal-trng.h
nrf51.h
aes.h arc4.h ascon.h asn.h asn_public.h async.h blake2-impl.h blake2-int.h blake2.h camellia.h chacha.h chacha20_poly1305.h cmac.h coding.h compress.h cpuid.h cryptocb.h curve25519.h curve448.h des3.h dh.h dilithium.h dsa.h ecc.h eccsi.h ed25519.h ed448.h error-crypt.h falcon.h fe_448.h fe_operations.h fips_test.h ge_448.h ge_operations.h hash.h hmac.h hpke.h include.am integer.h kdf.h libwolfssl_sources.h libwolfssl_sources_asm.h logging.h md2.h md4.h md5.h mem_track.h memory.h misc.h mpi_class.h mpi_superclass.h oid_sum.h pkcs11.h pkcs12.h pkcs7.h poly1305.h puf.h pwdbased.h random.h rc2.h ripemd.h rng_bank.h rsa.h sakke.h selftest.h settings.h sha.h sha256.h sha3.h sha512.h signature.h siphash.h sm2.h sm3.h sm4.h sp.h sp_int.h srp.h tfm.h types.h visibility.h wc_encrypt.h wc_lms.h wc_mlkem.h wc_pkcs11.h wc_port.h wc_she.h wc_slhdsa.h wc_xmss.h wolfentropy.h wolfevent.h wolfmath.h
callbacks.h certs_test.h certs_test_sm.h crl.h error-ssl.h include.am internal.h ocsp.h options.h.in quic.h sniffer.h sniffer_error.h sniffer_error.rc ssl.h test.h version.h version.h.in wolfio.h
wrapper
Ada
examples
src aes_verify_main.adb rsa_verify_main.adb sha256_main.adb spark_sockets.adb spark_sockets.ads spark_terminal.adb spark_terminal.ads tls_client.adb tls_client.ads tls_client_main.adb tls_server.adb tls_server.ads tls_server_main.adb
.gitignore alire.toml examples.gpr
tests
src
support test_support.adb test_support.ads tests_root_suite.adb tests_root_suite.ads
aes_bindings_tests.adb aes_bindings_tests.ads rsa_verify_bindings_tests.adb rsa_verify_bindings_tests.ads sha256_bindings_tests.adb sha256_bindings_tests.ads tests.adb
.gitignore README.md alire.toml tests.gpr valgrind.supp
.gitignore README.md ada_binding.c alire.toml default.gpr include.am restricted.adc user_settings.h wolfssl-full_runtime.adb wolfssl-full_runtime.ads wolfssl.adb wolfssl.ads wolfssl.gpr
CSharp
wolfCrypt-Test
Properties AssemblyInfo.cs
App.config wolfCrypt-Test.cs wolfCrypt-Test.csproj
wolfSSL-DTLS-PSK-Server
Properties AssemblyInfo.cs
App.config wolfSSL-DTLS-PSK-Server.cs wolfSSL-DTLS-PSK-Server.csproj
wolfSSL-DTLS-Server
Properties AssemblyInfo.cs
App.config wolfSSL-DTLS-Server.cs wolfSSL-DTLS-Server.csproj
wolfSSL-Example-IOCallbacks
Properties AssemblyInfo.cs
App.config wolfSSL-Example-IOCallbacks.cs wolfSSL-Example-IOCallbacks.csproj
wolfSSL-TLS-Client
Properties AssemblyInfo.cs
App.config wolfSSL-TLS-Client.cs wolfSSL-TLS-Client.csproj
wolfSSL-TLS-PSK-Client
Properties AssemblyInfo.cs
App.config wolfSSL-TLS-PSK-Client.cs wolfSSL-TLS-PSK-Client.csproj
wolfSSL-TLS-PSK-Server
Properties AssemblyInfo.cs
App.config wolfSSL-TLS-PSK-Server.cs wolfSSL-TLS-PSK-Server.csproj
wolfSSL-TLS-Server
Properties AssemblyInfo.cs
App.config wolfSSL-TLS-Server.cs wolfSSL-TLS-Server.csproj
wolfSSL-TLS-ServerThreaded
Properties AssemblyInfo.cs
App.config wolfSSL-TLS-ServerThreaded.cs wolfSSL-TLS-ServerThreaded.csproj
wolfSSL_CSharp
Properties AssemblyInfo.cs Resources.Designer.cs Resources.resx
X509.cs wolfCrypt.cs wolfSSL.cs wolfSSL_CSharp.csproj
README.md include.am user_settings.h wolfSSL_CSharp.sln wolfssl.vcxproj
python README.md
rust
wolfssl-wolfcrypt
src aes.rs blake2.rs chacha20_poly1305.rs cmac.rs cmac_mac.rs curve25519.rs dh.rs dilithium.rs ecc.rs ecdsa.rs ed25519.rs ed448.rs fips.rs hkdf.rs hmac.rs hmac_mac.rs kdf.rs lib.rs lms.rs mlkem.rs mlkem_kem.rs pbkdf2_password_hash.rs prf.rs random.rs rsa.rs rsa_pkcs1v15.rs sha.rs sha_digest.rs sys.rs
tests
common mod.rs
test_aes.rs test_blake2.rs test_chacha20_poly1305.rs test_cmac.rs test_cmac_mac.rs test_curve25519.rs test_dh.rs test_dilithium.rs test_ecc.rs test_ecdsa.rs test_ed25519.rs test_ed448.rs test_hkdf.rs test_hmac.rs test_hmac_mac.rs test_kdf.rs test_lms.rs test_mlkem.rs test_mlkem_kem.rs test_pbkdf2_password_hash.rs test_prf.rs test_random.rs test_rsa.rs test_rsa_pkcs1v15.rs test_sha.rs test_sha_digest.rs test_wolfcrypt.rs
CHANGELOG.md Cargo.lock Cargo.toml Makefile README.md build.rs headers.h
Makefile README.md include.am
include.am
zephyr
samples
wolfssl_benchmark
boards native_sim.conf nrf5340dk_nrf5340_cpuapp.conf nrf5340dk_nrf5340_cpuapp_ns.conf
CMakeLists.txt README install_test.sh prj.conf sample.yaml zephyr_legacy.conf zephyr_v4.1.conf
wolfssl_test
boards native_sim.conf nrf5340dk_nrf5340_cpuapp.conf nrf5340dk_nrf5340_cpuapp_ns.conf
CMakeLists.txt README install_test.sh prj-no-malloc.conf prj.conf sample.yaml zephyr_legacy.conf zephyr_v4.1.conf
wolfssl_tls_sock
boards native_sim.conf
src tls_sock.c
CMakeLists.txt README install_sample.sh prj-no-malloc.conf prj.conf sample.yaml zephyr_legacy.conf zephyr_v4.1.conf
wolfssl_tls_thread
boards native_sim.conf nrf5340dk_nrf5340_cpuapp.conf nrf5340dk_nrf5340_cpuapp_ns.conf
src tls_threaded.c
CMakeLists.txt README install_sample.sh prj.conf sample.yaml zephyr_legacy.conf zephyr_v4.1.conf
wolfssl options.h
CMakeLists.txt Kconfig Kconfig.tls-generic README.md include.am module.yml user_settings-no-malloc.h user_settings.h zephyr_init.c
.codespellexcludelines .cyignore .editorconfig .gitignore .wolfssl_known_macro_extras AUTHORS CMakeLists.txt CMakePresets.json CMakeSettings.json COPYING ChangeLog.md INSTALL LICENSING LPCExpresso.cproject LPCExpresso.project Makefile.am README README-async.md README.md SCRIPTS-LIST SECURITY-POLICY.md SECURITY-REPORT-TEMPLATE.md Vagrantfile autogen.sh commit-tests.sh configure.ac fips-check.sh fips-hash.sh gencertbuf.pl input pull_to_vagrant.sh quit resource.h stamp-h.in valgrind-bash.supp valgrind-error.sh wnr-example.conf wolfssl-VS2022.vcxproj wolfssl.rc wolfssl.vcproj wolfssl.vcxproj wolfssl64.sln
.clangd .gitignore DOCS.md Makefile README.md assert.c core.c crypto.c env.c fs.c http.c ini.c json.c log.c luna.h main.c makext.mk path.c process.c request.c sqlite.c stash.c template.c util.c
wolfssl/wolfcrypt/src/aes_gcm_asm.S raw 
    1/* aes_gcm_asm.S */
    2/*
    3 * Copyright (C) 2006-2026 wolfSSL Inc.
    4 *
    5 * This file is part of wolfSSL.
    6 *
    7 * wolfSSL is free software; you can redistribute it and/or modify
    8 * it under the terms of the GNU General Public License as published by
    9 * the Free Software Foundation; either version 3 of the License, or
   10 * (at your option) any later version.
   11 *
   12 * wolfSSL is distributed in the hope that it will be useful,
   13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
   14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   15 * GNU General Public License for more details.
   16 *
   17 * You should have received a copy of the GNU General Public License
   18 * along with this program; if not, write to the Free Software
   19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA
   20 */
   21
   22#ifdef WOLFSSL_USER_SETTINGS
   23#ifdef WOLFSSL_USER_SETTINGS_ASM
   24/*
   25 * user_settings_asm.h is a file generated by the script user_settings_asm.sh.
   26 * The script takes in a user_settings.h and produces user_settings_asm.h, which
   27 * is a stripped down version of user_settings.h containing only preprocessor
   28 * directives. This makes the header safe to include in assembly (.S) files.
   29 */
   30#include "user_settings_asm.h"
   31#else
   32/*
   33 * Note: if user_settings.h contains any C code (e.g. a typedef or function
   34 * prototype), including it here in an assembly (.S) file will cause an
   35 * assembler failure. See user_settings_asm.h above.
   36 */
   37#include "user_settings.h"
   38#endif /* WOLFSSL_USER_SETTINGS_ASM */
   39#endif /* WOLFSSL_USER_SETTINGS */
   40
   41#ifndef HAVE_INTEL_AVX1
   42#define HAVE_INTEL_AVX1
   43#endif /* HAVE_INTEL_AVX1 */
   44#ifndef NO_AVX2_SUPPORT
   45#ifndef HAVE_INTEL_AVX2
   46#define HAVE_INTEL_AVX2
   47#endif /* HAVE_INTEL_AVX2 */
   48#endif /* NO_AVX2_SUPPORT */
   49
   50#ifdef WOLFSSL_X86_64_BUILD
   51#ifndef __APPLE__
   52.data
   53#else
   54.section	__DATA,__data
   55#endif /* __APPLE__ */
   56#ifndef __APPLE__
   57.align	16
   58#else
   59.p2align	4
   60#endif /* __APPLE__ */
   61L_GCM_generate_m0_aesni_rev8:
   62.quad	0x08090a0b0c0d0e0f,0x0001020304050607
   63#ifndef __APPLE__
   64.data
   65#else
   66.section	__DATA,__data
   67#endif /* __APPLE__ */
   68#ifndef __APPLE__
   69.align	16
   70#else
   71.p2align	4
   72#endif /* __APPLE__ */
   73L_GCM_generate_m0_aesni_mod2_128:
   74.quad	0x0000000000000000,0xe100000000000000
   75#ifndef __APPLE__
   76.text
   77.globl	GCM_generate_m0_aesni
   78.type	GCM_generate_m0_aesni,@function
   79.align	16
   80GCM_generate_m0_aesni:
   81#else
   82.section	__TEXT,__text
   83.globl	_GCM_generate_m0_aesni
   84.p2align	4
   85_GCM_generate_m0_aesni:
   86#endif /* __APPLE__ */
   87        movdqu	L_GCM_generate_m0_aesni_rev8(%rip), %xmm9
   88        movdqu	L_GCM_generate_m0_aesni_mod2_128(%rip), %xmm10
   89        pxor	%xmm8, %xmm8
   90        movdqu	(%rdi), %xmm0
   91        movdqu	%xmm8, (%rsi)
   92        movdqu	%xmm0, %xmm8
   93        pshufb	%xmm9, %xmm0
   94        movdqu	%xmm0, %xmm5
   95        movdqu	%xmm0, %xmm4
   96        psllq	$63, %xmm5
   97        psrlq	$0x01, %xmm4
   98        movdqu	%xmm5, %xmm1
   99        pslldq	$8, %xmm1
  100        psrldq	$8, %xmm5
  101        pshufd	$0xff, %xmm1, %xmm1
  102        por	%xmm5, %xmm4
  103        psrad	$31, %xmm1
  104        pand	%xmm10, %xmm1
  105        pxor	%xmm4, %xmm1
  106        movdqu	%xmm1, %xmm5
  107        movdqu	%xmm1, %xmm4
  108        psllq	$63, %xmm5
  109        psrlq	$0x01, %xmm4
  110        movdqu	%xmm5, %xmm2
  111        pslldq	$8, %xmm2
  112        psrldq	$8, %xmm5
  113        pshufd	$0xff, %xmm2, %xmm2
  114        por	%xmm5, %xmm4
  115        psrad	$31, %xmm2
  116        pand	%xmm10, %xmm2
  117        pxor	%xmm4, %xmm2
  118        movdqu	%xmm2, %xmm5
  119        movdqu	%xmm2, %xmm4
  120        psllq	$63, %xmm5
  121        psrlq	$0x01, %xmm4
  122        movdqu	%xmm5, %xmm3
  123        pslldq	$8, %xmm3
  124        psrldq	$8, %xmm5
  125        pshufd	$0xff, %xmm3, %xmm3
  126        por	%xmm5, %xmm4
  127        psrad	$31, %xmm3
  128        pand	%xmm10, %xmm3
  129        pxor	%xmm4, %xmm3
  130        pshufb	%xmm9, %xmm3
  131        pshufb	%xmm9, %xmm2
  132        movdqu	%xmm3, %xmm8
  133        pshufb	%xmm9, %xmm1
  134        pshufb	%xmm9, %xmm0
  135        pxor	%xmm2, %xmm8
  136        movdqu	%xmm3, 16(%rsi)
  137        movdqu	%xmm2, 32(%rsi)
  138        movdqu	%xmm8, 48(%rsi)
  139        movdqu	%xmm1, 64(%rsi)
  140        movdqu	%xmm3, %xmm4
  141        movdqu	%xmm2, %xmm5
  142        movdqu	%xmm8, %xmm6
  143        pxor	%xmm1, %xmm4
  144        pxor	%xmm1, %xmm5
  145        pxor	%xmm1, %xmm6
  146        movdqu	%xmm4, 80(%rsi)
  147        movdqu	%xmm5, 96(%rsi)
  148        movdqu	%xmm6, 112(%rsi)
  149        movdqu	%xmm0, 128(%rsi)
  150        pxor	%xmm0, %xmm1
  151        movdqu	%xmm3, %xmm4
  152        movdqu	%xmm2, %xmm6
  153        pxor	%xmm0, %xmm4
  154        pxor	%xmm0, %xmm6
  155        movdqu	%xmm4, 144(%rsi)
  156        movdqu	%xmm6, 160(%rsi)
  157        pxor	%xmm3, %xmm6
  158        movdqu	%xmm6, 176(%rsi)
  159        movdqu	%xmm1, 192(%rsi)
  160        movdqu	%xmm3, %xmm4
  161        movdqu	%xmm2, %xmm5
  162        movdqu	%xmm8, %xmm6
  163        pxor	%xmm1, %xmm4
  164        pxor	%xmm1, %xmm5
  165        pxor	%xmm1, %xmm6
  166        movdqu	%xmm4, 208(%rsi)
  167        movdqu	%xmm5, 224(%rsi)
  168        movdqu	%xmm6, 240(%rsi)
  169        movdqu	(%rsi), %xmm0
  170        movdqu	16(%rsi), %xmm1
  171        movdqu	32(%rsi), %xmm2
  172        movdqu	48(%rsi), %xmm3
  173        pshufb	%xmm9, %xmm0
  174        pshufb	%xmm9, %xmm1
  175        pshufb	%xmm9, %xmm2
  176        pshufb	%xmm9, %xmm3
  177        movdqu	%xmm0, %xmm4
  178        movdqu	%xmm1, %xmm5
  179        movdqu	%xmm2, %xmm6
  180        movdqu	%xmm3, %xmm7
  181        psllq	$60, %xmm4
  182        psllq	$60, %xmm5
  183        psllq	$60, %xmm6
  184        psllq	$60, %xmm7
  185        psrlq	$4, %xmm0
  186        psrlq	$4, %xmm1
  187        psrlq	$4, %xmm2
  188        psrlq	$4, %xmm3
  189        psrldq	$8, %xmm4
  190        psrldq	$8, %xmm5
  191        psrldq	$8, %xmm6
  192        psrldq	$8, %xmm7
  193        por	%xmm4, %xmm0
  194        por	%xmm5, %xmm1
  195        por	%xmm6, %xmm2
  196        por	%xmm7, %xmm3
  197        vpshufb	%xmm9, %xmm0, %xmm0
  198        vpshufb	%xmm9, %xmm1, %xmm1
  199        vpshufb	%xmm9, %xmm2, %xmm2
  200        vpshufb	%xmm9, %xmm3, %xmm3
  201        movdqu	%xmm0, 256(%rsi)
  202        movdqu	%xmm1, 272(%rsi)
  203        movdqu	%xmm2, 288(%rsi)
  204        movdqu	%xmm3, 304(%rsi)
  205        movdqu	64(%rsi), %xmm0
  206        movdqu	80(%rsi), %xmm1
  207        movdqu	96(%rsi), %xmm2
  208        movdqu	112(%rsi), %xmm3
  209        pshufb	%xmm9, %xmm0
  210        pshufb	%xmm9, %xmm1
  211        pshufb	%xmm9, %xmm2
  212        pshufb	%xmm9, %xmm3
  213        movdqu	%xmm0, %xmm4
  214        movdqu	%xmm1, %xmm5
  215        movdqu	%xmm2, %xmm6
  216        movdqu	%xmm3, %xmm7
  217        psllq	$60, %xmm4
  218        psllq	$60, %xmm5
  219        psllq	$60, %xmm6
  220        psllq	$60, %xmm7
  221        psrlq	$4, %xmm0
  222        psrlq	$4, %xmm1
  223        psrlq	$4, %xmm2
  224        psrlq	$4, %xmm3
  225        psrldq	$8, %xmm4
  226        psrldq	$8, %xmm5
  227        psrldq	$8, %xmm6
  228        psrldq	$8, %xmm7
  229        por	%xmm4, %xmm0
  230        por	%xmm5, %xmm1
  231        por	%xmm6, %xmm2
  232        por	%xmm7, %xmm3
  233        vpshufb	%xmm9, %xmm0, %xmm0
  234        vpshufb	%xmm9, %xmm1, %xmm1
  235        vpshufb	%xmm9, %xmm2, %xmm2
  236        vpshufb	%xmm9, %xmm3, %xmm3
  237        movdqu	%xmm0, 320(%rsi)
  238        movdqu	%xmm1, 336(%rsi)
  239        movdqu	%xmm2, 352(%rsi)
  240        movdqu	%xmm3, 368(%rsi)
  241        movdqu	128(%rsi), %xmm0
  242        movdqu	144(%rsi), %xmm1
  243        movdqu	160(%rsi), %xmm2
  244        movdqu	176(%rsi), %xmm3
  245        pshufb	%xmm9, %xmm0
  246        pshufb	%xmm9, %xmm1
  247        pshufb	%xmm9, %xmm2
  248        pshufb	%xmm9, %xmm3
  249        movdqu	%xmm0, %xmm4
  250        movdqu	%xmm1, %xmm5
  251        movdqu	%xmm2, %xmm6
  252        movdqu	%xmm3, %xmm7
  253        psllq	$60, %xmm4
  254        psllq	$60, %xmm5
  255        psllq	$60, %xmm6
  256        psllq	$60, %xmm7
  257        psrlq	$4, %xmm0
  258        psrlq	$4, %xmm1
  259        psrlq	$4, %xmm2
  260        psrlq	$4, %xmm3
  261        psrldq	$8, %xmm4
  262        psrldq	$8, %xmm5
  263        psrldq	$8, %xmm6
  264        psrldq	$8, %xmm7
  265        por	%xmm4, %xmm0
  266        por	%xmm5, %xmm1
  267        por	%xmm6, %xmm2
  268        por	%xmm7, %xmm3
  269        vpshufb	%xmm9, %xmm0, %xmm0
  270        vpshufb	%xmm9, %xmm1, %xmm1
  271        vpshufb	%xmm9, %xmm2, %xmm2
  272        vpshufb	%xmm9, %xmm3, %xmm3
  273        movdqu	%xmm0, 384(%rsi)
  274        movdqu	%xmm1, 400(%rsi)
  275        movdqu	%xmm2, 416(%rsi)
  276        movdqu	%xmm3, 432(%rsi)
  277        movdqu	192(%rsi), %xmm0
  278        movdqu	208(%rsi), %xmm1
  279        movdqu	224(%rsi), %xmm2
  280        movdqu	240(%rsi), %xmm3
  281        pshufb	%xmm9, %xmm0
  282        pshufb	%xmm9, %xmm1
  283        pshufb	%xmm9, %xmm2
  284        pshufb	%xmm9, %xmm3
  285        movdqu	%xmm0, %xmm4
  286        movdqu	%xmm1, %xmm5
  287        movdqu	%xmm2, %xmm6
  288        movdqu	%xmm3, %xmm7
  289        psllq	$60, %xmm4
  290        psllq	$60, %xmm5
  291        psllq	$60, %xmm6
  292        psllq	$60, %xmm7
  293        psrlq	$4, %xmm0
  294        psrlq	$4, %xmm1
  295        psrlq	$4, %xmm2
  296        psrlq	$4, %xmm3
  297        psrldq	$8, %xmm4
  298        psrldq	$8, %xmm5
  299        psrldq	$8, %xmm6
  300        psrldq	$8, %xmm7
  301        por	%xmm4, %xmm0
  302        por	%xmm5, %xmm1
  303        por	%xmm6, %xmm2
  304        por	%xmm7, %xmm3
  305        vpshufb	%xmm9, %xmm0, %xmm0
  306        vpshufb	%xmm9, %xmm1, %xmm1
  307        vpshufb	%xmm9, %xmm2, %xmm2
  308        vpshufb	%xmm9, %xmm3, %xmm3
  309        movdqu	%xmm0, 448(%rsi)
  310        movdqu	%xmm1, 464(%rsi)
  311        movdqu	%xmm2, 480(%rsi)
  312        movdqu	%xmm3, 496(%rsi)
  313        repz retq
  314#ifndef __APPLE__
  315.size	GCM_generate_m0_aesni,.-GCM_generate_m0_aesni
  316#endif /* __APPLE__ */
  317#ifndef __APPLE__
  318.data
  319#else
  320.section	__DATA,__data
  321#endif /* __APPLE__ */
  322#ifndef __APPLE__
  323.align	16
  324#else
  325.p2align	4
  326#endif /* __APPLE__ */
  327L_aes_gcm_one:
  328.quad	0x0000000000000000,0x0000000000000001
  329#ifndef __APPLE__
  330.data
  331#else
  332.section	__DATA,__data
  333#endif /* __APPLE__ */
  334#ifndef __APPLE__
  335.align	16
  336#else
  337.p2align	4
  338#endif /* __APPLE__ */
  339L_aes_gcm_two:
  340.quad	0x0000000000000000,0x0000000000000002
  341#ifndef __APPLE__
  342.data
  343#else
  344.section	__DATA,__data
  345#endif /* __APPLE__ */
  346#ifndef __APPLE__
  347.align	16
  348#else
  349.p2align	4
  350#endif /* __APPLE__ */
  351L_aes_gcm_three:
  352.quad	0x0000000000000000,0x0000000000000003
  353#ifndef __APPLE__
  354.data
  355#else
  356.section	__DATA,__data
  357#endif /* __APPLE__ */
  358#ifndef __APPLE__
  359.align	16
  360#else
  361.p2align	4
  362#endif /* __APPLE__ */
  363L_aes_gcm_four:
  364.quad	0x0000000000000000,0x0000000000000004
  365#ifndef __APPLE__
  366.data
  367#else
  368.section	__DATA,__data
  369#endif /* __APPLE__ */
  370#ifndef __APPLE__
  371.align	16
  372#else
  373.p2align	4
  374#endif /* __APPLE__ */
  375L_aes_gcm_five:
  376.quad	0x0000000000000000,0x0000000000000005
  377#ifndef __APPLE__
  378.data
  379#else
  380.section	__DATA,__data
  381#endif /* __APPLE__ */
  382#ifndef __APPLE__
  383.align	16
  384#else
  385.p2align	4
  386#endif /* __APPLE__ */
  387L_aes_gcm_six:
  388.quad	0x0000000000000000,0x0000000000000006
  389#ifndef __APPLE__
  390.data
  391#else
  392.section	__DATA,__data
  393#endif /* __APPLE__ */
  394#ifndef __APPLE__
  395.align	16
  396#else
  397.p2align	4
  398#endif /* __APPLE__ */
  399L_aes_gcm_seven:
  400.quad	0x0000000000000000,0x0000000000000007
  401#ifndef __APPLE__
  402.data
  403#else
  404.section	__DATA,__data
  405#endif /* __APPLE__ */
  406#ifndef __APPLE__
  407.align	16
  408#else
  409.p2align	4
  410#endif /* __APPLE__ */
  411L_aes_gcm_eight:
  412.quad	0x0000000000000000,0x0000000000000008
  413#ifndef __APPLE__
  414.data
  415#else
  416.section	__DATA,__data
  417#endif /* __APPLE__ */
  418#ifndef __APPLE__
  419.align	16
  420#else
  421.p2align	4
  422#endif /* __APPLE__ */
  423L_aes_gcm_bswap_epi64:
  424.quad	0x0001020304050607,0x08090a0b0c0d0e0f
  425#ifndef __APPLE__
  426.data
  427#else
  428.section	__DATA,__data
  429#endif /* __APPLE__ */
  430#ifndef __APPLE__
  431.align	16
  432#else
  433.p2align	4
  434#endif /* __APPLE__ */
  435L_aes_gcm_bswap_mask:
  436.quad	0x08090a0b0c0d0e0f,0x0001020304050607
  437#ifndef __APPLE__
  438.data
  439#else
  440.section	__DATA,__data
  441#endif /* __APPLE__ */
  442#ifndef __APPLE__
  443.align	16
  444#else
  445.p2align	4
  446#endif /* __APPLE__ */
  447L_aes_gcm_mod2_128:
  448.quad	0x0000000000000001,0xc200000000000000
  449#ifndef __APPLE__
  450.text
  451.globl	AES_GCM_encrypt_aesni
  452.type	AES_GCM_encrypt_aesni,@function
  453.align	16
  454AES_GCM_encrypt_aesni:
  455#else
  456.section	__TEXT,__text
  457.globl	_AES_GCM_encrypt_aesni
  458.p2align	4
  459_AES_GCM_encrypt_aesni:
  460#endif /* __APPLE__ */
  461        pushq	%r13
  462        pushq	%r12
  463        pushq	%rbx
  464        pushq	%r14
  465        pushq	%r15
  466        movq	%rdx, %r12
  467        movq	%rcx, %rax
  468        movl	48(%rsp), %r11d
  469        movl	56(%rsp), %ebx
  470        movl	64(%rsp), %r14d
  471        movq	72(%rsp), %r15
  472        movl	80(%rsp), %r10d
  473        subq	$0xa0, %rsp
  474        pxor	%xmm4, %xmm4
  475        pxor	%xmm6, %xmm6
  476        cmpl	$12, %ebx
  477        movl	%ebx, %edx
  478        jne	L_AES_GCM_encrypt_aesni_iv_not_12
  479        # # Calculate values when IV is 12 bytes
  480        # Set counter based on IV
  481        movl	$0x1000000, %ecx
  482        pinsrq	$0x00, (%rax), %xmm4
  483        pinsrd	$2, 8(%rax), %xmm4
  484        pinsrd	$3, %ecx, %xmm4
  485        # H = Encrypt X(=0) and T = Encrypt counter
  486        movdqa	%xmm4, %xmm1
  487        movdqa	(%r15), %xmm5
  488        pxor	%xmm5, %xmm1
  489        movdqa	16(%r15), %xmm7
  490        aesenc	%xmm7, %xmm5
  491        aesenc	%xmm7, %xmm1
  492        movdqa	32(%r15), %xmm7
  493        aesenc	%xmm7, %xmm5
  494        aesenc	%xmm7, %xmm1
  495        movdqa	48(%r15), %xmm7
  496        aesenc	%xmm7, %xmm5
  497        aesenc	%xmm7, %xmm1
  498        movdqa	64(%r15), %xmm7
  499        aesenc	%xmm7, %xmm5
  500        aesenc	%xmm7, %xmm1
  501        movdqa	80(%r15), %xmm7
  502        aesenc	%xmm7, %xmm5
  503        aesenc	%xmm7, %xmm1
  504        movdqa	96(%r15), %xmm7
  505        aesenc	%xmm7, %xmm5
  506        aesenc	%xmm7, %xmm1
  507        movdqa	112(%r15), %xmm7
  508        aesenc	%xmm7, %xmm5
  509        aesenc	%xmm7, %xmm1
  510        movdqa	128(%r15), %xmm7
  511        aesenc	%xmm7, %xmm5
  512        aesenc	%xmm7, %xmm1
  513        movdqa	144(%r15), %xmm7
  514        aesenc	%xmm7, %xmm5
  515        aesenc	%xmm7, %xmm1
  516        cmpl	$11, %r10d
  517        movdqa	160(%r15), %xmm7
  518        jl	L_AES_GCM_encrypt_aesni_calc_iv_12_last
  519        aesenc	%xmm7, %xmm5
  520        aesenc	%xmm7, %xmm1
  521        movdqa	176(%r15), %xmm7
  522        aesenc	%xmm7, %xmm5
  523        aesenc	%xmm7, %xmm1
  524        cmpl	$13, %r10d
  525        movdqa	192(%r15), %xmm7
  526        jl	L_AES_GCM_encrypt_aesni_calc_iv_12_last
  527        aesenc	%xmm7, %xmm5
  528        aesenc	%xmm7, %xmm1
  529        movdqa	208(%r15), %xmm7
  530        aesenc	%xmm7, %xmm5
  531        aesenc	%xmm7, %xmm1
  532        movdqa	224(%r15), %xmm7
  533L_AES_GCM_encrypt_aesni_calc_iv_12_last:
  534        aesenclast	%xmm7, %xmm5
  535        aesenclast	%xmm7, %xmm1
  536        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm5
  537        movdqu	%xmm1, 144(%rsp)
  538        jmp	L_AES_GCM_encrypt_aesni_iv_done
  539L_AES_GCM_encrypt_aesni_iv_not_12:
  540        # Calculate values when IV is not 12 bytes
  541        # H = Encrypt X(=0)
  542        movdqa	(%r15), %xmm5
  543        aesenc	16(%r15), %xmm5
  544        aesenc	32(%r15), %xmm5
  545        aesenc	48(%r15), %xmm5
  546        aesenc	64(%r15), %xmm5
  547        aesenc	80(%r15), %xmm5
  548        aesenc	96(%r15), %xmm5
  549        aesenc	112(%r15), %xmm5
  550        aesenc	128(%r15), %xmm5
  551        aesenc	144(%r15), %xmm5
  552        cmpl	$11, %r10d
  553        movdqa	160(%r15), %xmm9
  554        jl	L_AES_GCM_encrypt_aesni_calc_iv_1_aesenc_avx_last
  555        aesenc	%xmm9, %xmm5
  556        aesenc	176(%r15), %xmm5
  557        cmpl	$13, %r10d
  558        movdqa	192(%r15), %xmm9
  559        jl	L_AES_GCM_encrypt_aesni_calc_iv_1_aesenc_avx_last
  560        aesenc	%xmm9, %xmm5
  561        aesenc	208(%r15), %xmm5
  562        movdqa	224(%r15), %xmm9
  563L_AES_GCM_encrypt_aesni_calc_iv_1_aesenc_avx_last:
  564        aesenclast	%xmm9, %xmm5
  565        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm5
  566        # Calc counter
  567        # Initialization vector
  568        cmpl	$0x00, %edx
  569        movq	$0x00, %rcx
  570        je	L_AES_GCM_encrypt_aesni_calc_iv_done
  571        cmpl	$16, %edx
  572        jl	L_AES_GCM_encrypt_aesni_calc_iv_lt16
  573        andl	$0xfffffff0, %edx
  574L_AES_GCM_encrypt_aesni_calc_iv_16_loop:
  575        movdqu	(%rax,%rcx,1), %xmm8
  576        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm8
  577        pxor	%xmm8, %xmm4
  578        pshufd	$0x4e, %xmm4, %xmm1
  579        pshufd	$0x4e, %xmm5, %xmm2
  580        movdqa	%xmm5, %xmm3
  581        movdqa	%xmm5, %xmm0
  582        pclmulqdq	$0x11, %xmm4, %xmm3
  583        pclmulqdq	$0x00, %xmm4, %xmm0
  584        pxor	%xmm4, %xmm1
  585        pxor	%xmm5, %xmm2
  586        pclmulqdq	$0x00, %xmm2, %xmm1
  587        pxor	%xmm0, %xmm1
  588        pxor	%xmm3, %xmm1
  589        movdqa	%xmm1, %xmm2
  590        movdqa	%xmm0, %xmm7
  591        movdqa	%xmm3, %xmm4
  592        pslldq	$8, %xmm2
  593        psrldq	$8, %xmm1
  594        pxor	%xmm2, %xmm7
  595        pxor	%xmm1, %xmm4
  596        movdqa	%xmm7, %xmm0
  597        movdqa	%xmm4, %xmm1
  598        psrld	$31, %xmm0
  599        psrld	$31, %xmm1
  600        pslld	$0x01, %xmm7
  601        pslld	$0x01, %xmm4
  602        movdqa	%xmm0, %xmm2
  603        pslldq	$4, %xmm0
  604        psrldq	$12, %xmm2
  605        pslldq	$4, %xmm1
  606        por	%xmm2, %xmm4
  607        por	%xmm0, %xmm7
  608        por	%xmm1, %xmm4
  609        movdqa	%xmm7, %xmm0
  610        movdqa	%xmm7, %xmm1
  611        movdqa	%xmm7, %xmm2
  612        pslld	$31, %xmm0
  613        pslld	$30, %xmm1
  614        pslld	$25, %xmm2
  615        pxor	%xmm1, %xmm0
  616        pxor	%xmm2, %xmm0
  617        movdqa	%xmm0, %xmm1
  618        psrldq	$4, %xmm1
  619        pslldq	$12, %xmm0
  620        pxor	%xmm0, %xmm7
  621        movdqa	%xmm7, %xmm2
  622        movdqa	%xmm7, %xmm3
  623        movdqa	%xmm7, %xmm0
  624        psrld	$0x01, %xmm2
  625        psrld	$2, %xmm3
  626        psrld	$7, %xmm0
  627        pxor	%xmm3, %xmm2
  628        pxor	%xmm0, %xmm2
  629        pxor	%xmm1, %xmm2
  630        pxor	%xmm7, %xmm2
  631        pxor	%xmm2, %xmm4
  632        addl	$16, %ecx
  633        cmpl	%edx, %ecx
  634        jl	L_AES_GCM_encrypt_aesni_calc_iv_16_loop
  635        movl	%ebx, %edx
  636        cmpl	%edx, %ecx
  637        je	L_AES_GCM_encrypt_aesni_calc_iv_done
  638L_AES_GCM_encrypt_aesni_calc_iv_lt16:
  639        subq	$16, %rsp
  640        pxor	%xmm8, %xmm8
  641        xorl	%ebx, %ebx
  642        movdqu	%xmm8, (%rsp)
  643L_AES_GCM_encrypt_aesni_calc_iv_loop:
  644        movzbl	(%rax,%rcx,1), %r13d
  645        movb	%r13b, (%rsp,%rbx,1)
  646        incl	%ecx
  647        incl	%ebx
  648        cmpl	%edx, %ecx
  649        jl	L_AES_GCM_encrypt_aesni_calc_iv_loop
  650        movdqu	(%rsp), %xmm8
  651        addq	$16, %rsp
  652        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm8
  653        pxor	%xmm8, %xmm4
  654        pshufd	$0x4e, %xmm4, %xmm1
  655        pshufd	$0x4e, %xmm5, %xmm2
  656        movdqa	%xmm5, %xmm3
  657        movdqa	%xmm5, %xmm0
  658        pclmulqdq	$0x11, %xmm4, %xmm3
  659        pclmulqdq	$0x00, %xmm4, %xmm0
  660        pxor	%xmm4, %xmm1
  661        pxor	%xmm5, %xmm2
  662        pclmulqdq	$0x00, %xmm2, %xmm1
  663        pxor	%xmm0, %xmm1
  664        pxor	%xmm3, %xmm1
  665        movdqa	%xmm1, %xmm2
  666        movdqa	%xmm0, %xmm7
  667        movdqa	%xmm3, %xmm4
  668        pslldq	$8, %xmm2
  669        psrldq	$8, %xmm1
  670        pxor	%xmm2, %xmm7
  671        pxor	%xmm1, %xmm4
  672        movdqa	%xmm7, %xmm0
  673        movdqa	%xmm4, %xmm1
  674        psrld	$31, %xmm0
  675        psrld	$31, %xmm1
  676        pslld	$0x01, %xmm7
  677        pslld	$0x01, %xmm4
  678        movdqa	%xmm0, %xmm2
  679        pslldq	$4, %xmm0
  680        psrldq	$12, %xmm2
  681        pslldq	$4, %xmm1
  682        por	%xmm2, %xmm4
  683        por	%xmm0, %xmm7
  684        por	%xmm1, %xmm4
  685        movdqa	%xmm7, %xmm0
  686        movdqa	%xmm7, %xmm1
  687        movdqa	%xmm7, %xmm2
  688        pslld	$31, %xmm0
  689        pslld	$30, %xmm1
  690        pslld	$25, %xmm2
  691        pxor	%xmm1, %xmm0
  692        pxor	%xmm2, %xmm0
  693        movdqa	%xmm0, %xmm1
  694        psrldq	$4, %xmm1
  695        pslldq	$12, %xmm0
  696        pxor	%xmm0, %xmm7
  697        movdqa	%xmm7, %xmm2
  698        movdqa	%xmm7, %xmm3
  699        movdqa	%xmm7, %xmm0
  700        psrld	$0x01, %xmm2
  701        psrld	$2, %xmm3
  702        psrld	$7, %xmm0
  703        pxor	%xmm3, %xmm2
  704        pxor	%xmm0, %xmm2
  705        pxor	%xmm1, %xmm2
  706        pxor	%xmm7, %xmm2
  707        pxor	%xmm2, %xmm4
  708L_AES_GCM_encrypt_aesni_calc_iv_done:
  709        # T = Encrypt counter
  710        pxor	%xmm0, %xmm0
  711        shll	$3, %edx
  712        pinsrq	$0x00, %rdx, %xmm0
  713        pxor	%xmm0, %xmm4
  714        pshufd	$0x4e, %xmm4, %xmm1
  715        pshufd	$0x4e, %xmm5, %xmm2
  716        movdqa	%xmm5, %xmm3
  717        movdqa	%xmm5, %xmm0
  718        pclmulqdq	$0x11, %xmm4, %xmm3
  719        pclmulqdq	$0x00, %xmm4, %xmm0
  720        pxor	%xmm4, %xmm1
  721        pxor	%xmm5, %xmm2
  722        pclmulqdq	$0x00, %xmm2, %xmm1
  723        pxor	%xmm0, %xmm1
  724        pxor	%xmm3, %xmm1
  725        movdqa	%xmm1, %xmm2
  726        movdqa	%xmm0, %xmm7
  727        movdqa	%xmm3, %xmm4
  728        pslldq	$8, %xmm2
  729        psrldq	$8, %xmm1
  730        pxor	%xmm2, %xmm7
  731        pxor	%xmm1, %xmm4
  732        movdqa	%xmm7, %xmm0
  733        movdqa	%xmm4, %xmm1
  734        psrld	$31, %xmm0
  735        psrld	$31, %xmm1
  736        pslld	$0x01, %xmm7
  737        pslld	$0x01, %xmm4
  738        movdqa	%xmm0, %xmm2
  739        pslldq	$4, %xmm0
  740        psrldq	$12, %xmm2
  741        pslldq	$4, %xmm1
  742        por	%xmm2, %xmm4
  743        por	%xmm0, %xmm7
  744        por	%xmm1, %xmm4
  745        movdqa	%xmm7, %xmm0
  746        movdqa	%xmm7, %xmm1
  747        movdqa	%xmm7, %xmm2
  748        pslld	$31, %xmm0
  749        pslld	$30, %xmm1
  750        pslld	$25, %xmm2
  751        pxor	%xmm1, %xmm0
  752        pxor	%xmm2, %xmm0
  753        movdqa	%xmm0, %xmm1
  754        psrldq	$4, %xmm1
  755        pslldq	$12, %xmm0
  756        pxor	%xmm0, %xmm7
  757        movdqa	%xmm7, %xmm2
  758        movdqa	%xmm7, %xmm3
  759        movdqa	%xmm7, %xmm0
  760        psrld	$0x01, %xmm2
  761        psrld	$2, %xmm3
  762        psrld	$7, %xmm0
  763        pxor	%xmm3, %xmm2
  764        pxor	%xmm0, %xmm2
  765        pxor	%xmm1, %xmm2
  766        pxor	%xmm7, %xmm2
  767        pxor	%xmm2, %xmm4
  768        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm4
  769        #   Encrypt counter
  770        movdqa	(%r15), %xmm8
  771        pxor	%xmm4, %xmm8
  772        aesenc	16(%r15), %xmm8
  773        aesenc	32(%r15), %xmm8
  774        aesenc	48(%r15), %xmm8
  775        aesenc	64(%r15), %xmm8
  776        aesenc	80(%r15), %xmm8
  777        aesenc	96(%r15), %xmm8
  778        aesenc	112(%r15), %xmm8
  779        aesenc	128(%r15), %xmm8
  780        aesenc	144(%r15), %xmm8
  781        cmpl	$11, %r10d
  782        movdqa	160(%r15), %xmm9
  783        jl	L_AES_GCM_encrypt_aesni_calc_iv_2_aesenc_avx_last
  784        aesenc	%xmm9, %xmm8
  785        aesenc	176(%r15), %xmm8
  786        cmpl	$13, %r10d
  787        movdqa	192(%r15), %xmm9
  788        jl	L_AES_GCM_encrypt_aesni_calc_iv_2_aesenc_avx_last
  789        aesenc	%xmm9, %xmm8
  790        aesenc	208(%r15), %xmm8
  791        movdqa	224(%r15), %xmm9
  792L_AES_GCM_encrypt_aesni_calc_iv_2_aesenc_avx_last:
  793        aesenclast	%xmm9, %xmm8
  794        movdqu	%xmm8, 144(%rsp)
  795L_AES_GCM_encrypt_aesni_iv_done:
  796        # Additional authentication data
  797        movl	%r11d, %edx
  798        cmpl	$0x00, %edx
  799        je	L_AES_GCM_encrypt_aesni_calc_aad_done
  800        xorl	%ecx, %ecx
  801        cmpl	$16, %edx
  802        jl	L_AES_GCM_encrypt_aesni_calc_aad_lt16
  803        andl	$0xfffffff0, %edx
  804L_AES_GCM_encrypt_aesni_calc_aad_16_loop:
  805        movdqu	(%r12,%rcx,1), %xmm8
  806        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm8
  807        pxor	%xmm8, %xmm6
  808        pshufd	$0x4e, %xmm6, %xmm1
  809        pshufd	$0x4e, %xmm5, %xmm2
  810        movdqa	%xmm5, %xmm3
  811        movdqa	%xmm5, %xmm0
  812        pclmulqdq	$0x11, %xmm6, %xmm3
  813        pclmulqdq	$0x00, %xmm6, %xmm0
  814        pxor	%xmm6, %xmm1
  815        pxor	%xmm5, %xmm2
  816        pclmulqdq	$0x00, %xmm2, %xmm1
  817        pxor	%xmm0, %xmm1
  818        pxor	%xmm3, %xmm1
  819        movdqa	%xmm1, %xmm2
  820        movdqa	%xmm0, %xmm7
  821        movdqa	%xmm3, %xmm6
  822        pslldq	$8, %xmm2
  823        psrldq	$8, %xmm1
  824        pxor	%xmm2, %xmm7
  825        pxor	%xmm1, %xmm6
  826        movdqa	%xmm7, %xmm0
  827        movdqa	%xmm6, %xmm1
  828        psrld	$31, %xmm0
  829        psrld	$31, %xmm1
  830        pslld	$0x01, %xmm7
  831        pslld	$0x01, %xmm6
  832        movdqa	%xmm0, %xmm2
  833        pslldq	$4, %xmm0
  834        psrldq	$12, %xmm2
  835        pslldq	$4, %xmm1
  836        por	%xmm2, %xmm6
  837        por	%xmm0, %xmm7
  838        por	%xmm1, %xmm6
  839        movdqa	%xmm7, %xmm0
  840        movdqa	%xmm7, %xmm1
  841        movdqa	%xmm7, %xmm2
  842        pslld	$31, %xmm0
  843        pslld	$30, %xmm1
  844        pslld	$25, %xmm2
  845        pxor	%xmm1, %xmm0
  846        pxor	%xmm2, %xmm0
  847        movdqa	%xmm0, %xmm1
  848        psrldq	$4, %xmm1
  849        pslldq	$12, %xmm0
  850        pxor	%xmm0, %xmm7
  851        movdqa	%xmm7, %xmm2
  852        movdqa	%xmm7, %xmm3
  853        movdqa	%xmm7, %xmm0
  854        psrld	$0x01, %xmm2
  855        psrld	$2, %xmm3
  856        psrld	$7, %xmm0
  857        pxor	%xmm3, %xmm2
  858        pxor	%xmm0, %xmm2
  859        pxor	%xmm1, %xmm2
  860        pxor	%xmm7, %xmm2
  861        pxor	%xmm2, %xmm6
  862        addl	$16, %ecx
  863        cmpl	%edx, %ecx
  864        jl	L_AES_GCM_encrypt_aesni_calc_aad_16_loop
  865        movl	%r11d, %edx
  866        cmpl	%edx, %ecx
  867        je	L_AES_GCM_encrypt_aesni_calc_aad_done
  868L_AES_GCM_encrypt_aesni_calc_aad_lt16:
  869        subq	$16, %rsp
  870        pxor	%xmm8, %xmm8
  871        xorl	%ebx, %ebx
  872        movdqu	%xmm8, (%rsp)
  873L_AES_GCM_encrypt_aesni_calc_aad_loop:
  874        movzbl	(%r12,%rcx,1), %r13d
  875        movb	%r13b, (%rsp,%rbx,1)
  876        incl	%ecx
  877        incl	%ebx
  878        cmpl	%edx, %ecx
  879        jl	L_AES_GCM_encrypt_aesni_calc_aad_loop
  880        movdqu	(%rsp), %xmm8
  881        addq	$16, %rsp
  882        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm8
  883        pxor	%xmm8, %xmm6
  884        pshufd	$0x4e, %xmm6, %xmm1
  885        pshufd	$0x4e, %xmm5, %xmm2
  886        movdqa	%xmm5, %xmm3
  887        movdqa	%xmm5, %xmm0
  888        pclmulqdq	$0x11, %xmm6, %xmm3
  889        pclmulqdq	$0x00, %xmm6, %xmm0
  890        pxor	%xmm6, %xmm1
  891        pxor	%xmm5, %xmm2
  892        pclmulqdq	$0x00, %xmm2, %xmm1
  893        pxor	%xmm0, %xmm1
  894        pxor	%xmm3, %xmm1
  895        movdqa	%xmm1, %xmm2
  896        movdqa	%xmm0, %xmm7
  897        movdqa	%xmm3, %xmm6
  898        pslldq	$8, %xmm2
  899        psrldq	$8, %xmm1
  900        pxor	%xmm2, %xmm7
  901        pxor	%xmm1, %xmm6
  902        movdqa	%xmm7, %xmm0
  903        movdqa	%xmm6, %xmm1
  904        psrld	$31, %xmm0
  905        psrld	$31, %xmm1
  906        pslld	$0x01, %xmm7
  907        pslld	$0x01, %xmm6
  908        movdqa	%xmm0, %xmm2
  909        pslldq	$4, %xmm0
  910        psrldq	$12, %xmm2
  911        pslldq	$4, %xmm1
  912        por	%xmm2, %xmm6
  913        por	%xmm0, %xmm7
  914        por	%xmm1, %xmm6
  915        movdqa	%xmm7, %xmm0
  916        movdqa	%xmm7, %xmm1
  917        movdqa	%xmm7, %xmm2
  918        pslld	$31, %xmm0
  919        pslld	$30, %xmm1
  920        pslld	$25, %xmm2
  921        pxor	%xmm1, %xmm0
  922        pxor	%xmm2, %xmm0
  923        movdqa	%xmm0, %xmm1
  924        psrldq	$4, %xmm1
  925        pslldq	$12, %xmm0
  926        pxor	%xmm0, %xmm7
  927        movdqa	%xmm7, %xmm2
  928        movdqa	%xmm7, %xmm3
  929        movdqa	%xmm7, %xmm0
  930        psrld	$0x01, %xmm2
  931        psrld	$2, %xmm3
  932        psrld	$7, %xmm0
  933        pxor	%xmm3, %xmm2
  934        pxor	%xmm0, %xmm2
  935        pxor	%xmm1, %xmm2
  936        pxor	%xmm7, %xmm2
  937        pxor	%xmm2, %xmm6
  938L_AES_GCM_encrypt_aesni_calc_aad_done:
  939        # Calculate counter and H
  940        pshufb	L_aes_gcm_bswap_epi64(%rip), %xmm4
  941        movdqa	%xmm5, %xmm9
  942        paddd	L_aes_gcm_one(%rip), %xmm4
  943        movdqa	%xmm5, %xmm8
  944        movdqu	%xmm4, 128(%rsp)
  945        psrlq	$63, %xmm9
  946        psllq	$0x01, %xmm8
  947        pslldq	$8, %xmm9
  948        por	%xmm9, %xmm8
  949        pshufd	$0xff, %xmm5, %xmm5
  950        psrad	$31, %xmm5
  951        pand	L_aes_gcm_mod2_128(%rip), %xmm5
  952        pxor	%xmm8, %xmm5
  953        xorq	%rbx, %rbx
  954        cmpl	$0x80, %r9d
  955        movl	%r9d, %r13d
  956        jl	L_AES_GCM_encrypt_aesni_done_128
  957        andl	$0xffffff80, %r13d
  958        movdqa	%xmm6, %xmm2
  959        # H ^ 1
  960        movdqu	%xmm5, (%rsp)
  961        # H ^ 2
  962        pshufd	$0x4e, %xmm5, %xmm9
  963        pshufd	$0x4e, %xmm5, %xmm10
  964        movdqa	%xmm5, %xmm11
  965        movdqa	%xmm5, %xmm8
  966        pclmulqdq	$0x11, %xmm5, %xmm11
  967        pclmulqdq	$0x00, %xmm5, %xmm8
  968        pxor	%xmm5, %xmm9
  969        pxor	%xmm5, %xmm10
  970        pclmulqdq	$0x00, %xmm10, %xmm9
  971        pxor	%xmm8, %xmm9
  972        pxor	%xmm11, %xmm9
  973        movdqa	%xmm9, %xmm10
  974        movdqa	%xmm11, %xmm0
  975        pslldq	$8, %xmm10
  976        psrldq	$8, %xmm9
  977        pxor	%xmm10, %xmm8
  978        pxor	%xmm9, %xmm0
  979        movdqa	%xmm8, %xmm12
  980        movdqa	%xmm8, %xmm13
  981        movdqa	%xmm8, %xmm14
  982        pslld	$31, %xmm12
  983        pslld	$30, %xmm13
  984        pslld	$25, %xmm14
  985        pxor	%xmm13, %xmm12
  986        pxor	%xmm14, %xmm12
  987        movdqa	%xmm12, %xmm13
  988        psrldq	$4, %xmm13
  989        pslldq	$12, %xmm12
  990        pxor	%xmm12, %xmm8
  991        movdqa	%xmm8, %xmm14
  992        movdqa	%xmm8, %xmm10
  993        movdqa	%xmm8, %xmm9
  994        psrld	$0x01, %xmm14
  995        psrld	$2, %xmm10
  996        psrld	$7, %xmm9
  997        pxor	%xmm10, %xmm14
  998        pxor	%xmm9, %xmm14
  999        pxor	%xmm13, %xmm14
 1000        pxor	%xmm8, %xmm14
 1001        pxor	%xmm14, %xmm0
 1002        movdqu	%xmm0, 16(%rsp)
 1003        # H ^ 3
 1004        pshufd	$0x4e, %xmm5, %xmm9
 1005        pshufd	$0x4e, %xmm0, %xmm10
 1006        movdqa	%xmm0, %xmm11
 1007        movdqa	%xmm0, %xmm8
 1008        pclmulqdq	$0x11, %xmm5, %xmm11
 1009        pclmulqdq	$0x00, %xmm5, %xmm8
 1010        pxor	%xmm5, %xmm9
 1011        pxor	%xmm0, %xmm10
 1012        pclmulqdq	$0x00, %xmm10, %xmm9
 1013        pxor	%xmm8, %xmm9
 1014        pxor	%xmm11, %xmm9
 1015        movdqa	%xmm9, %xmm10
 1016        movdqa	%xmm11, %xmm1
 1017        pslldq	$8, %xmm10
 1018        psrldq	$8, %xmm9
 1019        pxor	%xmm10, %xmm8
 1020        pxor	%xmm9, %xmm1
 1021        movdqa	%xmm8, %xmm12
 1022        movdqa	%xmm8, %xmm13
 1023        movdqa	%xmm8, %xmm14
 1024        pslld	$31, %xmm12
 1025        pslld	$30, %xmm13
 1026        pslld	$25, %xmm14
 1027        pxor	%xmm13, %xmm12
 1028        pxor	%xmm14, %xmm12
 1029        movdqa	%xmm12, %xmm13
 1030        psrldq	$4, %xmm13
 1031        pslldq	$12, %xmm12
 1032        pxor	%xmm12, %xmm8
 1033        movdqa	%xmm8, %xmm14
 1034        movdqa	%xmm8, %xmm10
 1035        movdqa	%xmm8, %xmm9
 1036        psrld	$0x01, %xmm14
 1037        psrld	$2, %xmm10
 1038        psrld	$7, %xmm9
 1039        pxor	%xmm10, %xmm14
 1040        pxor	%xmm9, %xmm14
 1041        pxor	%xmm13, %xmm14
 1042        pxor	%xmm8, %xmm14
 1043        pxor	%xmm14, %xmm1
 1044        movdqu	%xmm1, 32(%rsp)
 1045        # H ^ 4
 1046        pshufd	$0x4e, %xmm0, %xmm9
 1047        pshufd	$0x4e, %xmm0, %xmm10
 1048        movdqa	%xmm0, %xmm11
 1049        movdqa	%xmm0, %xmm8
 1050        pclmulqdq	$0x11, %xmm0, %xmm11
 1051        pclmulqdq	$0x00, %xmm0, %xmm8
 1052        pxor	%xmm0, %xmm9
 1053        pxor	%xmm0, %xmm10
 1054        pclmulqdq	$0x00, %xmm10, %xmm9
 1055        pxor	%xmm8, %xmm9
 1056        pxor	%xmm11, %xmm9
 1057        movdqa	%xmm9, %xmm10
 1058        movdqa	%xmm11, %xmm3
 1059        pslldq	$8, %xmm10
 1060        psrldq	$8, %xmm9
 1061        pxor	%xmm10, %xmm8
 1062        pxor	%xmm9, %xmm3
 1063        movdqa	%xmm8, %xmm12
 1064        movdqa	%xmm8, %xmm13
 1065        movdqa	%xmm8, %xmm14
 1066        pslld	$31, %xmm12
 1067        pslld	$30, %xmm13
 1068        pslld	$25, %xmm14
 1069        pxor	%xmm13, %xmm12
 1070        pxor	%xmm14, %xmm12
 1071        movdqa	%xmm12, %xmm13
 1072        psrldq	$4, %xmm13
 1073        pslldq	$12, %xmm12
 1074        pxor	%xmm12, %xmm8
 1075        movdqa	%xmm8, %xmm14
 1076        movdqa	%xmm8, %xmm10
 1077        movdqa	%xmm8, %xmm9
 1078        psrld	$0x01, %xmm14
 1079        psrld	$2, %xmm10
 1080        psrld	$7, %xmm9
 1081        pxor	%xmm10, %xmm14
 1082        pxor	%xmm9, %xmm14
 1083        pxor	%xmm13, %xmm14
 1084        pxor	%xmm8, %xmm14
 1085        pxor	%xmm14, %xmm3
 1086        movdqu	%xmm3, 48(%rsp)
 1087        # H ^ 5
 1088        pshufd	$0x4e, %xmm0, %xmm9
 1089        pshufd	$0x4e, %xmm1, %xmm10
 1090        movdqa	%xmm1, %xmm11
 1091        movdqa	%xmm1, %xmm8
 1092        pclmulqdq	$0x11, %xmm0, %xmm11
 1093        pclmulqdq	$0x00, %xmm0, %xmm8
 1094        pxor	%xmm0, %xmm9
 1095        pxor	%xmm1, %xmm10
 1096        pclmulqdq	$0x00, %xmm10, %xmm9
 1097        pxor	%xmm8, %xmm9
 1098        pxor	%xmm11, %xmm9
 1099        movdqa	%xmm9, %xmm10
 1100        movdqa	%xmm11, %xmm7
 1101        pslldq	$8, %xmm10
 1102        psrldq	$8, %xmm9
 1103        pxor	%xmm10, %xmm8
 1104        pxor	%xmm9, %xmm7
 1105        movdqa	%xmm8, %xmm12
 1106        movdqa	%xmm8, %xmm13
 1107        movdqa	%xmm8, %xmm14
 1108        pslld	$31, %xmm12
 1109        pslld	$30, %xmm13
 1110        pslld	$25, %xmm14
 1111        pxor	%xmm13, %xmm12
 1112        pxor	%xmm14, %xmm12
 1113        movdqa	%xmm12, %xmm13
 1114        psrldq	$4, %xmm13
 1115        pslldq	$12, %xmm12
 1116        pxor	%xmm12, %xmm8
 1117        movdqa	%xmm8, %xmm14
 1118        movdqa	%xmm8, %xmm10
 1119        movdqa	%xmm8, %xmm9
 1120        psrld	$0x01, %xmm14
 1121        psrld	$2, %xmm10
 1122        psrld	$7, %xmm9
 1123        pxor	%xmm10, %xmm14
 1124        pxor	%xmm9, %xmm14
 1125        pxor	%xmm13, %xmm14
 1126        pxor	%xmm8, %xmm14
 1127        pxor	%xmm14, %xmm7
 1128        movdqu	%xmm7, 64(%rsp)
 1129        # H ^ 6
 1130        pshufd	$0x4e, %xmm1, %xmm9
 1131        pshufd	$0x4e, %xmm1, %xmm10
 1132        movdqa	%xmm1, %xmm11
 1133        movdqa	%xmm1, %xmm8
 1134        pclmulqdq	$0x11, %xmm1, %xmm11
 1135        pclmulqdq	$0x00, %xmm1, %xmm8
 1136        pxor	%xmm1, %xmm9
 1137        pxor	%xmm1, %xmm10
 1138        pclmulqdq	$0x00, %xmm10, %xmm9
 1139        pxor	%xmm8, %xmm9
 1140        pxor	%xmm11, %xmm9
 1141        movdqa	%xmm9, %xmm10
 1142        movdqa	%xmm11, %xmm7
 1143        pslldq	$8, %xmm10
 1144        psrldq	$8, %xmm9
 1145        pxor	%xmm10, %xmm8
 1146        pxor	%xmm9, %xmm7
 1147        movdqa	%xmm8, %xmm12
 1148        movdqa	%xmm8, %xmm13
 1149        movdqa	%xmm8, %xmm14
 1150        pslld	$31, %xmm12
 1151        pslld	$30, %xmm13
 1152        pslld	$25, %xmm14
 1153        pxor	%xmm13, %xmm12
 1154        pxor	%xmm14, %xmm12
 1155        movdqa	%xmm12, %xmm13
 1156        psrldq	$4, %xmm13
 1157        pslldq	$12, %xmm12
 1158        pxor	%xmm12, %xmm8
 1159        movdqa	%xmm8, %xmm14
 1160        movdqa	%xmm8, %xmm10
 1161        movdqa	%xmm8, %xmm9
 1162        psrld	$0x01, %xmm14
 1163        psrld	$2, %xmm10
 1164        psrld	$7, %xmm9
 1165        pxor	%xmm10, %xmm14
 1166        pxor	%xmm9, %xmm14
 1167        pxor	%xmm13, %xmm14
 1168        pxor	%xmm8, %xmm14
 1169        pxor	%xmm14, %xmm7
 1170        movdqu	%xmm7, 80(%rsp)
 1171        # H ^ 7
 1172        pshufd	$0x4e, %xmm1, %xmm9
 1173        pshufd	$0x4e, %xmm3, %xmm10
 1174        movdqa	%xmm3, %xmm11
 1175        movdqa	%xmm3, %xmm8
 1176        pclmulqdq	$0x11, %xmm1, %xmm11
 1177        pclmulqdq	$0x00, %xmm1, %xmm8
 1178        pxor	%xmm1, %xmm9
 1179        pxor	%xmm3, %xmm10
 1180        pclmulqdq	$0x00, %xmm10, %xmm9
 1181        pxor	%xmm8, %xmm9
 1182        pxor	%xmm11, %xmm9
 1183        movdqa	%xmm9, %xmm10
 1184        movdqa	%xmm11, %xmm7
 1185        pslldq	$8, %xmm10
 1186        psrldq	$8, %xmm9
 1187        pxor	%xmm10, %xmm8
 1188        pxor	%xmm9, %xmm7
 1189        movdqa	%xmm8, %xmm12
 1190        movdqa	%xmm8, %xmm13
 1191        movdqa	%xmm8, %xmm14
 1192        pslld	$31, %xmm12
 1193        pslld	$30, %xmm13
 1194        pslld	$25, %xmm14
 1195        pxor	%xmm13, %xmm12
 1196        pxor	%xmm14, %xmm12
 1197        movdqa	%xmm12, %xmm13
 1198        psrldq	$4, %xmm13
 1199        pslldq	$12, %xmm12
 1200        pxor	%xmm12, %xmm8
 1201        movdqa	%xmm8, %xmm14
 1202        movdqa	%xmm8, %xmm10
 1203        movdqa	%xmm8, %xmm9
 1204        psrld	$0x01, %xmm14
 1205        psrld	$2, %xmm10
 1206        psrld	$7, %xmm9
 1207        pxor	%xmm10, %xmm14
 1208        pxor	%xmm9, %xmm14
 1209        pxor	%xmm13, %xmm14
 1210        pxor	%xmm8, %xmm14
 1211        pxor	%xmm14, %xmm7
 1212        movdqu	%xmm7, 96(%rsp)
 1213        # H ^ 8
 1214        pshufd	$0x4e, %xmm3, %xmm9
 1215        pshufd	$0x4e, %xmm3, %xmm10
 1216        movdqa	%xmm3, %xmm11
 1217        movdqa	%xmm3, %xmm8
 1218        pclmulqdq	$0x11, %xmm3, %xmm11
 1219        pclmulqdq	$0x00, %xmm3, %xmm8
 1220        pxor	%xmm3, %xmm9
 1221        pxor	%xmm3, %xmm10
 1222        pclmulqdq	$0x00, %xmm10, %xmm9
 1223        pxor	%xmm8, %xmm9
 1224        pxor	%xmm11, %xmm9
 1225        movdqa	%xmm9, %xmm10
 1226        movdqa	%xmm11, %xmm7
 1227        pslldq	$8, %xmm10
 1228        psrldq	$8, %xmm9
 1229        pxor	%xmm10, %xmm8
 1230        pxor	%xmm9, %xmm7
 1231        movdqa	%xmm8, %xmm12
 1232        movdqa	%xmm8, %xmm13
 1233        movdqa	%xmm8, %xmm14
 1234        pslld	$31, %xmm12
 1235        pslld	$30, %xmm13
 1236        pslld	$25, %xmm14
 1237        pxor	%xmm13, %xmm12
 1238        pxor	%xmm14, %xmm12
 1239        movdqa	%xmm12, %xmm13
 1240        psrldq	$4, %xmm13
 1241        pslldq	$12, %xmm12
 1242        pxor	%xmm12, %xmm8
 1243        movdqa	%xmm8, %xmm14
 1244        movdqa	%xmm8, %xmm10
 1245        movdqa	%xmm8, %xmm9
 1246        psrld	$0x01, %xmm14
 1247        psrld	$2, %xmm10
 1248        psrld	$7, %xmm9
 1249        pxor	%xmm10, %xmm14
 1250        pxor	%xmm9, %xmm14
 1251        pxor	%xmm13, %xmm14
 1252        pxor	%xmm8, %xmm14
 1253        pxor	%xmm14, %xmm7
 1254        movdqu	%xmm7, 112(%rsp)
 1255        # First 128 bytes of input
 1256        movdqu	128(%rsp), %xmm8
 1257        movdqa	L_aes_gcm_bswap_epi64(%rip), %xmm1
 1258        movdqa	%xmm8, %xmm0
 1259        pshufb	%xmm1, %xmm8
 1260        movdqa	%xmm0, %xmm9
 1261        paddd	L_aes_gcm_one(%rip), %xmm9
 1262        pshufb	%xmm1, %xmm9
 1263        movdqa	%xmm0, %xmm10
 1264        paddd	L_aes_gcm_two(%rip), %xmm10
 1265        pshufb	%xmm1, %xmm10
 1266        movdqa	%xmm0, %xmm11
 1267        paddd	L_aes_gcm_three(%rip), %xmm11
 1268        pshufb	%xmm1, %xmm11
 1269        movdqa	%xmm0, %xmm12
 1270        paddd	L_aes_gcm_four(%rip), %xmm12
 1271        pshufb	%xmm1, %xmm12
 1272        movdqa	%xmm0, %xmm13
 1273        paddd	L_aes_gcm_five(%rip), %xmm13
 1274        pshufb	%xmm1, %xmm13
 1275        movdqa	%xmm0, %xmm14
 1276        paddd	L_aes_gcm_six(%rip), %xmm14
 1277        pshufb	%xmm1, %xmm14
 1278        movdqa	%xmm0, %xmm15
 1279        paddd	L_aes_gcm_seven(%rip), %xmm15
 1280        pshufb	%xmm1, %xmm15
 1281        paddd	L_aes_gcm_eight(%rip), %xmm0
 1282        movdqa	(%r15), %xmm7
 1283        movdqu	%xmm0, 128(%rsp)
 1284        pxor	%xmm7, %xmm8
 1285        pxor	%xmm7, %xmm9
 1286        pxor	%xmm7, %xmm10
 1287        pxor	%xmm7, %xmm11
 1288        pxor	%xmm7, %xmm12
 1289        pxor	%xmm7, %xmm13
 1290        pxor	%xmm7, %xmm14
 1291        pxor	%xmm7, %xmm15
 1292        movdqa	16(%r15), %xmm7
 1293        aesenc	%xmm7, %xmm8
 1294        aesenc	%xmm7, %xmm9
 1295        aesenc	%xmm7, %xmm10
 1296        aesenc	%xmm7, %xmm11
 1297        aesenc	%xmm7, %xmm12
 1298        aesenc	%xmm7, %xmm13
 1299        aesenc	%xmm7, %xmm14
 1300        aesenc	%xmm7, %xmm15
 1301        movdqa	32(%r15), %xmm7
 1302        aesenc	%xmm7, %xmm8
 1303        aesenc	%xmm7, %xmm9
 1304        aesenc	%xmm7, %xmm10
 1305        aesenc	%xmm7, %xmm11
 1306        aesenc	%xmm7, %xmm12
 1307        aesenc	%xmm7, %xmm13
 1308        aesenc	%xmm7, %xmm14
 1309        aesenc	%xmm7, %xmm15
 1310        movdqa	48(%r15), %xmm7
 1311        aesenc	%xmm7, %xmm8
 1312        aesenc	%xmm7, %xmm9
 1313        aesenc	%xmm7, %xmm10
 1314        aesenc	%xmm7, %xmm11
 1315        aesenc	%xmm7, %xmm12
 1316        aesenc	%xmm7, %xmm13
 1317        aesenc	%xmm7, %xmm14
 1318        aesenc	%xmm7, %xmm15
 1319        movdqa	64(%r15), %xmm7
 1320        aesenc	%xmm7, %xmm8
 1321        aesenc	%xmm7, %xmm9
 1322        aesenc	%xmm7, %xmm10
 1323        aesenc	%xmm7, %xmm11
 1324        aesenc	%xmm7, %xmm12
 1325        aesenc	%xmm7, %xmm13
 1326        aesenc	%xmm7, %xmm14
 1327        aesenc	%xmm7, %xmm15
 1328        movdqa	80(%r15), %xmm7
 1329        aesenc	%xmm7, %xmm8
 1330        aesenc	%xmm7, %xmm9
 1331        aesenc	%xmm7, %xmm10
 1332        aesenc	%xmm7, %xmm11
 1333        aesenc	%xmm7, %xmm12
 1334        aesenc	%xmm7, %xmm13
 1335        aesenc	%xmm7, %xmm14
 1336        aesenc	%xmm7, %xmm15
 1337        movdqa	96(%r15), %xmm7
 1338        aesenc	%xmm7, %xmm8
 1339        aesenc	%xmm7, %xmm9
 1340        aesenc	%xmm7, %xmm10
 1341        aesenc	%xmm7, %xmm11
 1342        aesenc	%xmm7, %xmm12
 1343        aesenc	%xmm7, %xmm13
 1344        aesenc	%xmm7, %xmm14
 1345        aesenc	%xmm7, %xmm15
 1346        movdqa	112(%r15), %xmm7
 1347        aesenc	%xmm7, %xmm8
 1348        aesenc	%xmm7, %xmm9
 1349        aesenc	%xmm7, %xmm10
 1350        aesenc	%xmm7, %xmm11
 1351        aesenc	%xmm7, %xmm12
 1352        aesenc	%xmm7, %xmm13
 1353        aesenc	%xmm7, %xmm14
 1354        aesenc	%xmm7, %xmm15
 1355        movdqa	128(%r15), %xmm7
 1356        aesenc	%xmm7, %xmm8
 1357        aesenc	%xmm7, %xmm9
 1358        aesenc	%xmm7, %xmm10
 1359        aesenc	%xmm7, %xmm11
 1360        aesenc	%xmm7, %xmm12
 1361        aesenc	%xmm7, %xmm13
 1362        aesenc	%xmm7, %xmm14
 1363        aesenc	%xmm7, %xmm15
 1364        movdqa	144(%r15), %xmm7
 1365        aesenc	%xmm7, %xmm8
 1366        aesenc	%xmm7, %xmm9
 1367        aesenc	%xmm7, %xmm10
 1368        aesenc	%xmm7, %xmm11
 1369        aesenc	%xmm7, %xmm12
 1370        aesenc	%xmm7, %xmm13
 1371        aesenc	%xmm7, %xmm14
 1372        aesenc	%xmm7, %xmm15
 1373        cmpl	$11, %r10d
 1374        movdqa	160(%r15), %xmm7
 1375        jl	L_AES_GCM_encrypt_aesni_enc_done
 1376        aesenc	%xmm7, %xmm8
 1377        aesenc	%xmm7, %xmm9
 1378        aesenc	%xmm7, %xmm10
 1379        aesenc	%xmm7, %xmm11
 1380        aesenc	%xmm7, %xmm12
 1381        aesenc	%xmm7, %xmm13
 1382        aesenc	%xmm7, %xmm14
 1383        aesenc	%xmm7, %xmm15
 1384        movdqa	176(%r15), %xmm7
 1385        aesenc	%xmm7, %xmm8
 1386        aesenc	%xmm7, %xmm9
 1387        aesenc	%xmm7, %xmm10
 1388        aesenc	%xmm7, %xmm11
 1389        aesenc	%xmm7, %xmm12
 1390        aesenc	%xmm7, %xmm13
 1391        aesenc	%xmm7, %xmm14
 1392        aesenc	%xmm7, %xmm15
 1393        cmpl	$13, %r10d
 1394        movdqa	192(%r15), %xmm7
 1395        jl	L_AES_GCM_encrypt_aesni_enc_done
 1396        aesenc	%xmm7, %xmm8
 1397        aesenc	%xmm7, %xmm9
 1398        aesenc	%xmm7, %xmm10
 1399        aesenc	%xmm7, %xmm11
 1400        aesenc	%xmm7, %xmm12
 1401        aesenc	%xmm7, %xmm13
 1402        aesenc	%xmm7, %xmm14
 1403        aesenc	%xmm7, %xmm15
 1404        movdqa	208(%r15), %xmm7
 1405        aesenc	%xmm7, %xmm8
 1406        aesenc	%xmm7, %xmm9
 1407        aesenc	%xmm7, %xmm10
 1408        aesenc	%xmm7, %xmm11
 1409        aesenc	%xmm7, %xmm12
 1410        aesenc	%xmm7, %xmm13
 1411        aesenc	%xmm7, %xmm14
 1412        aesenc	%xmm7, %xmm15
 1413        movdqa	224(%r15), %xmm7
 1414L_AES_GCM_encrypt_aesni_enc_done:
 1415        aesenclast	%xmm7, %xmm8
 1416        aesenclast	%xmm7, %xmm9
 1417        movdqu	(%rdi), %xmm0
 1418        movdqu	16(%rdi), %xmm1
 1419        pxor	%xmm0, %xmm8
 1420        pxor	%xmm1, %xmm9
 1421        movdqu	%xmm8, (%rsi)
 1422        movdqu	%xmm9, 16(%rsi)
 1423        aesenclast	%xmm7, %xmm10
 1424        aesenclast	%xmm7, %xmm11
 1425        movdqu	32(%rdi), %xmm0
 1426        movdqu	48(%rdi), %xmm1
 1427        pxor	%xmm0, %xmm10
 1428        pxor	%xmm1, %xmm11
 1429        movdqu	%xmm10, 32(%rsi)
 1430        movdqu	%xmm11, 48(%rsi)
 1431        aesenclast	%xmm7, %xmm12
 1432        aesenclast	%xmm7, %xmm13
 1433        movdqu	64(%rdi), %xmm0
 1434        movdqu	80(%rdi), %xmm1
 1435        pxor	%xmm0, %xmm12
 1436        pxor	%xmm1, %xmm13
 1437        movdqu	%xmm12, 64(%rsi)
 1438        movdqu	%xmm13, 80(%rsi)
 1439        aesenclast	%xmm7, %xmm14
 1440        aesenclast	%xmm7, %xmm15
 1441        movdqu	96(%rdi), %xmm0
 1442        movdqu	112(%rdi), %xmm1
 1443        pxor	%xmm0, %xmm14
 1444        pxor	%xmm1, %xmm15
 1445        movdqu	%xmm14, 96(%rsi)
 1446        movdqu	%xmm15, 112(%rsi)
 1447        cmpl	$0x80, %r13d
 1448        movl	$0x80, %ebx
 1449        jle	L_AES_GCM_encrypt_aesni_end_128
 1450        # More 128 bytes of input
 1451L_AES_GCM_encrypt_aesni_ghash_128:
 1452        leaq	(%rdi,%rbx,1), %rcx
 1453        leaq	(%rsi,%rbx,1), %rdx
 1454        movdqu	128(%rsp), %xmm8
 1455        movdqa	L_aes_gcm_bswap_epi64(%rip), %xmm1
 1456        movdqa	%xmm8, %xmm0
 1457        pshufb	%xmm1, %xmm8
 1458        movdqa	%xmm0, %xmm9
 1459        paddd	L_aes_gcm_one(%rip), %xmm9
 1460        pshufb	%xmm1, %xmm9
 1461        movdqa	%xmm0, %xmm10
 1462        paddd	L_aes_gcm_two(%rip), %xmm10
 1463        pshufb	%xmm1, %xmm10
 1464        movdqa	%xmm0, %xmm11
 1465        paddd	L_aes_gcm_three(%rip), %xmm11
 1466        pshufb	%xmm1, %xmm11
 1467        movdqa	%xmm0, %xmm12
 1468        paddd	L_aes_gcm_four(%rip), %xmm12
 1469        pshufb	%xmm1, %xmm12
 1470        movdqa	%xmm0, %xmm13
 1471        paddd	L_aes_gcm_five(%rip), %xmm13
 1472        pshufb	%xmm1, %xmm13
 1473        movdqa	%xmm0, %xmm14
 1474        paddd	L_aes_gcm_six(%rip), %xmm14
 1475        pshufb	%xmm1, %xmm14
 1476        movdqa	%xmm0, %xmm15
 1477        paddd	L_aes_gcm_seven(%rip), %xmm15
 1478        pshufb	%xmm1, %xmm15
 1479        paddd	L_aes_gcm_eight(%rip), %xmm0
 1480        movdqa	(%r15), %xmm7
 1481        movdqu	%xmm0, 128(%rsp)
 1482        pxor	%xmm7, %xmm8
 1483        pxor	%xmm7, %xmm9
 1484        pxor	%xmm7, %xmm10
 1485        pxor	%xmm7, %xmm11
 1486        pxor	%xmm7, %xmm12
 1487        pxor	%xmm7, %xmm13
 1488        pxor	%xmm7, %xmm14
 1489        pxor	%xmm7, %xmm15
 1490        movdqu	112(%rsp), %xmm7
 1491        movdqu	-128(%rdx), %xmm0
 1492        aesenc	16(%r15), %xmm8
 1493        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm0
 1494        pxor	%xmm2, %xmm0
 1495        pshufd	$0x4e, %xmm7, %xmm1
 1496        pshufd	$0x4e, %xmm0, %xmm5
 1497        pxor	%xmm7, %xmm1
 1498        pxor	%xmm0, %xmm5
 1499        movdqa	%xmm0, %xmm3
 1500        pclmulqdq	$0x11, %xmm7, %xmm3
 1501        aesenc	16(%r15), %xmm9
 1502        aesenc	16(%r15), %xmm10
 1503        movdqa	%xmm0, %xmm2
 1504        pclmulqdq	$0x00, %xmm7, %xmm2
 1505        aesenc	16(%r15), %xmm11
 1506        aesenc	16(%r15), %xmm12
 1507        pclmulqdq	$0x00, %xmm5, %xmm1
 1508        aesenc	16(%r15), %xmm13
 1509        aesenc	16(%r15), %xmm14
 1510        aesenc	16(%r15), %xmm15
 1511        pxor	%xmm2, %xmm1
 1512        pxor	%xmm3, %xmm1
 1513        movdqu	96(%rsp), %xmm7
 1514        movdqu	-112(%rdx), %xmm0
 1515        pshufd	$0x4e, %xmm7, %xmm4
 1516        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm0
 1517        aesenc	32(%r15), %xmm8
 1518        pxor	%xmm7, %xmm4
 1519        pshufd	$0x4e, %xmm0, %xmm5
 1520        pxor	%xmm0, %xmm5
 1521        movdqa	%xmm0, %xmm6
 1522        pclmulqdq	$0x11, %xmm7, %xmm6
 1523        aesenc	32(%r15), %xmm9
 1524        aesenc	32(%r15), %xmm10
 1525        pclmulqdq	$0x00, %xmm0, %xmm7
 1526        aesenc	32(%r15), %xmm11
 1527        aesenc	32(%r15), %xmm12
 1528        pclmulqdq	$0x00, %xmm5, %xmm4
 1529        aesenc	32(%r15), %xmm13
 1530        aesenc	32(%r15), %xmm14
 1531        aesenc	32(%r15), %xmm15
 1532        pxor	%xmm7, %xmm1
 1533        pxor	%xmm7, %xmm2
 1534        pxor	%xmm6, %xmm1
 1535        pxor	%xmm6, %xmm3
 1536        pxor	%xmm4, %xmm1
 1537        movdqu	80(%rsp), %xmm7
 1538        movdqu	-96(%rdx), %xmm0
 1539        pshufd	$0x4e, %xmm7, %xmm4
 1540        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm0
 1541        aesenc	48(%r15), %xmm8
 1542        pxor	%xmm7, %xmm4
 1543        pshufd	$0x4e, %xmm0, %xmm5
 1544        pxor	%xmm0, %xmm5
 1545        movdqa	%xmm0, %xmm6
 1546        pclmulqdq	$0x11, %xmm7, %xmm6
 1547        aesenc	48(%r15), %xmm9
 1548        aesenc	48(%r15), %xmm10
 1549        pclmulqdq	$0x00, %xmm0, %xmm7
 1550        aesenc	48(%r15), %xmm11
 1551        aesenc	48(%r15), %xmm12
 1552        pclmulqdq	$0x00, %xmm5, %xmm4
 1553        aesenc	48(%r15), %xmm13
 1554        aesenc	48(%r15), %xmm14
 1555        aesenc	48(%r15), %xmm15
 1556        pxor	%xmm7, %xmm1
 1557        pxor	%xmm7, %xmm2
 1558        pxor	%xmm6, %xmm1
 1559        pxor	%xmm6, %xmm3
 1560        pxor	%xmm4, %xmm1
 1561        movdqu	64(%rsp), %xmm7
 1562        movdqu	-80(%rdx), %xmm0
 1563        pshufd	$0x4e, %xmm7, %xmm4
 1564        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm0
 1565        aesenc	64(%r15), %xmm8
 1566        pxor	%xmm7, %xmm4
 1567        pshufd	$0x4e, %xmm0, %xmm5
 1568        pxor	%xmm0, %xmm5
 1569        movdqa	%xmm0, %xmm6
 1570        pclmulqdq	$0x11, %xmm7, %xmm6
 1571        aesenc	64(%r15), %xmm9
 1572        aesenc	64(%r15), %xmm10
 1573        pclmulqdq	$0x00, %xmm0, %xmm7
 1574        aesenc	64(%r15), %xmm11
 1575        aesenc	64(%r15), %xmm12
 1576        pclmulqdq	$0x00, %xmm5, %xmm4
 1577        aesenc	64(%r15), %xmm13
 1578        aesenc	64(%r15), %xmm14
 1579        aesenc	64(%r15), %xmm15
 1580        pxor	%xmm7, %xmm1
 1581        pxor	%xmm7, %xmm2
 1582        pxor	%xmm6, %xmm1
 1583        pxor	%xmm6, %xmm3
 1584        pxor	%xmm4, %xmm1
 1585        movdqu	48(%rsp), %xmm7
 1586        movdqu	-64(%rdx), %xmm0
 1587        pshufd	$0x4e, %xmm7, %xmm4
 1588        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm0
 1589        aesenc	80(%r15), %xmm8
 1590        pxor	%xmm7, %xmm4
 1591        pshufd	$0x4e, %xmm0, %xmm5
 1592        pxor	%xmm0, %xmm5
 1593        movdqa	%xmm0, %xmm6
 1594        pclmulqdq	$0x11, %xmm7, %xmm6
 1595        aesenc	80(%r15), %xmm9
 1596        aesenc	80(%r15), %xmm10
 1597        pclmulqdq	$0x00, %xmm0, %xmm7
 1598        aesenc	80(%r15), %xmm11
 1599        aesenc	80(%r15), %xmm12
 1600        pclmulqdq	$0x00, %xmm5, %xmm4
 1601        aesenc	80(%r15), %xmm13
 1602        aesenc	80(%r15), %xmm14
 1603        aesenc	80(%r15), %xmm15
 1604        pxor	%xmm7, %xmm1
 1605        pxor	%xmm7, %xmm2
 1606        pxor	%xmm6, %xmm1
 1607        pxor	%xmm6, %xmm3
 1608        pxor	%xmm4, %xmm1
 1609        movdqu	32(%rsp), %xmm7
 1610        movdqu	-48(%rdx), %xmm0
 1611        pshufd	$0x4e, %xmm7, %xmm4
 1612        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm0
 1613        aesenc	96(%r15), %xmm8
 1614        pxor	%xmm7, %xmm4
 1615        pshufd	$0x4e, %xmm0, %xmm5
 1616        pxor	%xmm0, %xmm5
 1617        movdqa	%xmm0, %xmm6
 1618        pclmulqdq	$0x11, %xmm7, %xmm6
 1619        aesenc	96(%r15), %xmm9
 1620        aesenc	96(%r15), %xmm10
 1621        pclmulqdq	$0x00, %xmm0, %xmm7
 1622        aesenc	96(%r15), %xmm11
 1623        aesenc	96(%r15), %xmm12
 1624        pclmulqdq	$0x00, %xmm5, %xmm4
 1625        aesenc	96(%r15), %xmm13
 1626        aesenc	96(%r15), %xmm14
 1627        aesenc	96(%r15), %xmm15
 1628        pxor	%xmm7, %xmm1
 1629        pxor	%xmm7, %xmm2
 1630        pxor	%xmm6, %xmm1
 1631        pxor	%xmm6, %xmm3
 1632        pxor	%xmm4, %xmm1
 1633        movdqu	16(%rsp), %xmm7
 1634        movdqu	-32(%rdx), %xmm0
 1635        pshufd	$0x4e, %xmm7, %xmm4
 1636        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm0
 1637        aesenc	112(%r15), %xmm8
 1638        pxor	%xmm7, %xmm4
 1639        pshufd	$0x4e, %xmm0, %xmm5
 1640        pxor	%xmm0, %xmm5
 1641        movdqa	%xmm0, %xmm6
 1642        pclmulqdq	$0x11, %xmm7, %xmm6
 1643        aesenc	112(%r15), %xmm9
 1644        aesenc	112(%r15), %xmm10
 1645        pclmulqdq	$0x00, %xmm0, %xmm7
 1646        aesenc	112(%r15), %xmm11
 1647        aesenc	112(%r15), %xmm12
 1648        pclmulqdq	$0x00, %xmm5, %xmm4
 1649        aesenc	112(%r15), %xmm13
 1650        aesenc	112(%r15), %xmm14
 1651        aesenc	112(%r15), %xmm15
 1652        pxor	%xmm7, %xmm1
 1653        pxor	%xmm7, %xmm2
 1654        pxor	%xmm6, %xmm1
 1655        pxor	%xmm6, %xmm3
 1656        pxor	%xmm4, %xmm1
 1657        movdqu	(%rsp), %xmm7
 1658        movdqu	-16(%rdx), %xmm0
 1659        pshufd	$0x4e, %xmm7, %xmm4
 1660        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm0
 1661        aesenc	128(%r15), %xmm8
 1662        pxor	%xmm7, %xmm4
 1663        pshufd	$0x4e, %xmm0, %xmm5
 1664        pxor	%xmm0, %xmm5
 1665        movdqa	%xmm0, %xmm6
 1666        pclmulqdq	$0x11, %xmm7, %xmm6
 1667        aesenc	128(%r15), %xmm9
 1668        aesenc	128(%r15), %xmm10
 1669        pclmulqdq	$0x00, %xmm0, %xmm7
 1670        aesenc	128(%r15), %xmm11
 1671        aesenc	128(%r15), %xmm12
 1672        pclmulqdq	$0x00, %xmm5, %xmm4
 1673        aesenc	128(%r15), %xmm13
 1674        aesenc	128(%r15), %xmm14
 1675        aesenc	128(%r15), %xmm15
 1676        pxor	%xmm7, %xmm1
 1677        pxor	%xmm7, %xmm2
 1678        pxor	%xmm6, %xmm1
 1679        pxor	%xmm6, %xmm3
 1680        pxor	%xmm4, %xmm1
 1681        movdqa	%xmm1, %xmm5
 1682        psrldq	$8, %xmm1
 1683        pslldq	$8, %xmm5
 1684        aesenc	144(%r15), %xmm8
 1685        pxor	%xmm5, %xmm2
 1686        pxor	%xmm1, %xmm3
 1687        movdqa	%xmm2, %xmm7
 1688        movdqa	%xmm2, %xmm4
 1689        movdqa	%xmm2, %xmm5
 1690        aesenc	144(%r15), %xmm9
 1691        pslld	$31, %xmm7
 1692        pslld	$30, %xmm4
 1693        pslld	$25, %xmm5
 1694        aesenc	144(%r15), %xmm10
 1695        pxor	%xmm4, %xmm7
 1696        pxor	%xmm5, %xmm7
 1697        aesenc	144(%r15), %xmm11
 1698        movdqa	%xmm7, %xmm4
 1699        pslldq	$12, %xmm7
 1700        psrldq	$4, %xmm4
 1701        aesenc	144(%r15), %xmm12
 1702        pxor	%xmm7, %xmm2
 1703        movdqa	%xmm2, %xmm5
 1704        movdqa	%xmm2, %xmm1
 1705        movdqa	%xmm2, %xmm0
 1706        aesenc	144(%r15), %xmm13
 1707        psrld	$0x01, %xmm5
 1708        psrld	$2, %xmm1
 1709        psrld	$7, %xmm0
 1710        aesenc	144(%r15), %xmm14
 1711        pxor	%xmm1, %xmm5
 1712        pxor	%xmm0, %xmm5
 1713        aesenc	144(%r15), %xmm15
 1714        pxor	%xmm4, %xmm5
 1715        pxor	%xmm5, %xmm2
 1716        pxor	%xmm3, %xmm2
 1717        cmpl	$11, %r10d
 1718        movdqa	160(%r15), %xmm7
 1719        jl	L_AES_GCM_encrypt_aesni_aesenc_128_ghash_avx_done
 1720        aesenc	%xmm7, %xmm8
 1721        aesenc	%xmm7, %xmm9
 1722        aesenc	%xmm7, %xmm10
 1723        aesenc	%xmm7, %xmm11
 1724        aesenc	%xmm7, %xmm12
 1725        aesenc	%xmm7, %xmm13
 1726        aesenc	%xmm7, %xmm14
 1727        aesenc	%xmm7, %xmm15
 1728        movdqa	176(%r15), %xmm7
 1729        aesenc	%xmm7, %xmm8
 1730        aesenc	%xmm7, %xmm9
 1731        aesenc	%xmm7, %xmm10
 1732        aesenc	%xmm7, %xmm11
 1733        aesenc	%xmm7, %xmm12
 1734        aesenc	%xmm7, %xmm13
 1735        aesenc	%xmm7, %xmm14
 1736        aesenc	%xmm7, %xmm15
 1737        cmpl	$13, %r10d
 1738        movdqa	192(%r15), %xmm7
 1739        jl	L_AES_GCM_encrypt_aesni_aesenc_128_ghash_avx_done
 1740        aesenc	%xmm7, %xmm8
 1741        aesenc	%xmm7, %xmm9
 1742        aesenc	%xmm7, %xmm10
 1743        aesenc	%xmm7, %xmm11
 1744        aesenc	%xmm7, %xmm12
 1745        aesenc	%xmm7, %xmm13
 1746        aesenc	%xmm7, %xmm14
 1747        aesenc	%xmm7, %xmm15
 1748        movdqa	208(%r15), %xmm7
 1749        aesenc	%xmm7, %xmm8
 1750        aesenc	%xmm7, %xmm9
 1751        aesenc	%xmm7, %xmm10
 1752        aesenc	%xmm7, %xmm11
 1753        aesenc	%xmm7, %xmm12
 1754        aesenc	%xmm7, %xmm13
 1755        aesenc	%xmm7, %xmm14
 1756        aesenc	%xmm7, %xmm15
 1757        movdqa	224(%r15), %xmm7
 1758L_AES_GCM_encrypt_aesni_aesenc_128_ghash_avx_done:
 1759        aesenclast	%xmm7, %xmm8
 1760        aesenclast	%xmm7, %xmm9
 1761        movdqu	(%rcx), %xmm0
 1762        movdqu	16(%rcx), %xmm1
 1763        pxor	%xmm0, %xmm8
 1764        pxor	%xmm1, %xmm9
 1765        movdqu	%xmm8, (%rdx)
 1766        movdqu	%xmm9, 16(%rdx)
 1767        aesenclast	%xmm7, %xmm10
 1768        aesenclast	%xmm7, %xmm11
 1769        movdqu	32(%rcx), %xmm0
 1770        movdqu	48(%rcx), %xmm1
 1771        pxor	%xmm0, %xmm10
 1772        pxor	%xmm1, %xmm11
 1773        movdqu	%xmm10, 32(%rdx)
 1774        movdqu	%xmm11, 48(%rdx)
 1775        aesenclast	%xmm7, %xmm12
 1776        aesenclast	%xmm7, %xmm13
 1777        movdqu	64(%rcx), %xmm0
 1778        movdqu	80(%rcx), %xmm1
 1779        pxor	%xmm0, %xmm12
 1780        pxor	%xmm1, %xmm13
 1781        movdqu	%xmm12, 64(%rdx)
 1782        movdqu	%xmm13, 80(%rdx)
 1783        aesenclast	%xmm7, %xmm14
 1784        aesenclast	%xmm7, %xmm15
 1785        movdqu	96(%rcx), %xmm0
 1786        movdqu	112(%rcx), %xmm1
 1787        pxor	%xmm0, %xmm14
 1788        pxor	%xmm1, %xmm15
 1789        movdqu	%xmm14, 96(%rdx)
 1790        movdqu	%xmm15, 112(%rdx)
 1791        addl	$0x80, %ebx
 1792        cmpl	%r13d, %ebx
 1793        jl	L_AES_GCM_encrypt_aesni_ghash_128
 1794L_AES_GCM_encrypt_aesni_end_128:
 1795        movdqa	L_aes_gcm_bswap_mask(%rip), %xmm4
 1796        pshufb	%xmm4, %xmm8
 1797        pshufb	%xmm4, %xmm9
 1798        pshufb	%xmm4, %xmm10
 1799        pshufb	%xmm4, %xmm11
 1800        pxor	%xmm2, %xmm8
 1801        pshufb	%xmm4, %xmm12
 1802        pshufb	%xmm4, %xmm13
 1803        pshufb	%xmm4, %xmm14
 1804        pshufb	%xmm4, %xmm15
 1805        movdqu	112(%rsp), %xmm7
 1806        pshufd	$0x4e, %xmm8, %xmm1
 1807        pshufd	$0x4e, %xmm7, %xmm2
 1808        movdqa	%xmm7, %xmm3
 1809        movdqa	%xmm7, %xmm0
 1810        pclmulqdq	$0x11, %xmm8, %xmm3
 1811        pclmulqdq	$0x00, %xmm8, %xmm0
 1812        pxor	%xmm8, %xmm1
 1813        pxor	%xmm7, %xmm2
 1814        pclmulqdq	$0x00, %xmm2, %xmm1
 1815        pxor	%xmm0, %xmm1
 1816        pxor	%xmm3, %xmm1
 1817        movdqa	%xmm1, %xmm2
 1818        movdqa	%xmm0, %xmm4
 1819        movdqa	%xmm3, %xmm6
 1820        pslldq	$8, %xmm2
 1821        psrldq	$8, %xmm1
 1822        pxor	%xmm2, %xmm4
 1823        pxor	%xmm1, %xmm6
 1824        movdqu	96(%rsp), %xmm7
 1825        pshufd	$0x4e, %xmm9, %xmm1
 1826        pshufd	$0x4e, %xmm7, %xmm2
 1827        movdqa	%xmm7, %xmm3
 1828        movdqa	%xmm7, %xmm0
 1829        pclmulqdq	$0x11, %xmm9, %xmm3
 1830        pclmulqdq	$0x00, %xmm9, %xmm0
 1831        pxor	%xmm9, %xmm1
 1832        pxor	%xmm7, %xmm2
 1833        pclmulqdq	$0x00, %xmm2, %xmm1
 1834        pxor	%xmm0, %xmm1
 1835        pxor	%xmm3, %xmm1
 1836        movdqa	%xmm1, %xmm2
 1837        pxor	%xmm0, %xmm4
 1838        pxor	%xmm3, %xmm6
 1839        pslldq	$8, %xmm2
 1840        psrldq	$8, %xmm1
 1841        pxor	%xmm2, %xmm4
 1842        pxor	%xmm1, %xmm6
 1843        movdqu	80(%rsp), %xmm7
 1844        pshufd	$0x4e, %xmm10, %xmm1
 1845        pshufd	$0x4e, %xmm7, %xmm2
 1846        movdqa	%xmm7, %xmm3
 1847        movdqa	%xmm7, %xmm0
 1848        pclmulqdq	$0x11, %xmm10, %xmm3
 1849        pclmulqdq	$0x00, %xmm10, %xmm0
 1850        pxor	%xmm10, %xmm1
 1851        pxor	%xmm7, %xmm2
 1852        pclmulqdq	$0x00, %xmm2, %xmm1
 1853        pxor	%xmm0, %xmm1
 1854        pxor	%xmm3, %xmm1
 1855        movdqa	%xmm1, %xmm2
 1856        pxor	%xmm0, %xmm4
 1857        pxor	%xmm3, %xmm6
 1858        pslldq	$8, %xmm2
 1859        psrldq	$8, %xmm1
 1860        pxor	%xmm2, %xmm4
 1861        pxor	%xmm1, %xmm6
 1862        movdqu	64(%rsp), %xmm7
 1863        pshufd	$0x4e, %xmm11, %xmm1
 1864        pshufd	$0x4e, %xmm7, %xmm2
 1865        movdqa	%xmm7, %xmm3
 1866        movdqa	%xmm7, %xmm0
 1867        pclmulqdq	$0x11, %xmm11, %xmm3
 1868        pclmulqdq	$0x00, %xmm11, %xmm0
 1869        pxor	%xmm11, %xmm1
 1870        pxor	%xmm7, %xmm2
 1871        pclmulqdq	$0x00, %xmm2, %xmm1
 1872        pxor	%xmm0, %xmm1
 1873        pxor	%xmm3, %xmm1
 1874        movdqa	%xmm1, %xmm2
 1875        pxor	%xmm0, %xmm4
 1876        pxor	%xmm3, %xmm6
 1877        pslldq	$8, %xmm2
 1878        psrldq	$8, %xmm1
 1879        pxor	%xmm2, %xmm4
 1880        pxor	%xmm1, %xmm6
 1881        movdqu	48(%rsp), %xmm7
 1882        pshufd	$0x4e, %xmm12, %xmm1
 1883        pshufd	$0x4e, %xmm7, %xmm2
 1884        movdqa	%xmm7, %xmm3
 1885        movdqa	%xmm7, %xmm0
 1886        pclmulqdq	$0x11, %xmm12, %xmm3
 1887        pclmulqdq	$0x00, %xmm12, %xmm0
 1888        pxor	%xmm12, %xmm1
 1889        pxor	%xmm7, %xmm2
 1890        pclmulqdq	$0x00, %xmm2, %xmm1
 1891        pxor	%xmm0, %xmm1
 1892        pxor	%xmm3, %xmm1
 1893        movdqa	%xmm1, %xmm2
 1894        pxor	%xmm0, %xmm4
 1895        pxor	%xmm3, %xmm6
 1896        pslldq	$8, %xmm2
 1897        psrldq	$8, %xmm1
 1898        pxor	%xmm2, %xmm4
 1899        pxor	%xmm1, %xmm6
 1900        movdqu	32(%rsp), %xmm7
 1901        pshufd	$0x4e, %xmm13, %xmm1
 1902        pshufd	$0x4e, %xmm7, %xmm2
 1903        movdqa	%xmm7, %xmm3
 1904        movdqa	%xmm7, %xmm0
 1905        pclmulqdq	$0x11, %xmm13, %xmm3
 1906        pclmulqdq	$0x00, %xmm13, %xmm0
 1907        pxor	%xmm13, %xmm1
 1908        pxor	%xmm7, %xmm2
 1909        pclmulqdq	$0x00, %xmm2, %xmm1
 1910        pxor	%xmm0, %xmm1
 1911        pxor	%xmm3, %xmm1
 1912        movdqa	%xmm1, %xmm2
 1913        pxor	%xmm0, %xmm4
 1914        pxor	%xmm3, %xmm6
 1915        pslldq	$8, %xmm2
 1916        psrldq	$8, %xmm1
 1917        pxor	%xmm2, %xmm4
 1918        pxor	%xmm1, %xmm6
 1919        movdqu	16(%rsp), %xmm7
 1920        pshufd	$0x4e, %xmm14, %xmm1
 1921        pshufd	$0x4e, %xmm7, %xmm2
 1922        movdqa	%xmm7, %xmm3
 1923        movdqa	%xmm7, %xmm0
 1924        pclmulqdq	$0x11, %xmm14, %xmm3
 1925        pclmulqdq	$0x00, %xmm14, %xmm0
 1926        pxor	%xmm14, %xmm1
 1927        pxor	%xmm7, %xmm2
 1928        pclmulqdq	$0x00, %xmm2, %xmm1
 1929        pxor	%xmm0, %xmm1
 1930        pxor	%xmm3, %xmm1
 1931        movdqa	%xmm1, %xmm2
 1932        pxor	%xmm0, %xmm4
 1933        pxor	%xmm3, %xmm6
 1934        pslldq	$8, %xmm2
 1935        psrldq	$8, %xmm1
 1936        pxor	%xmm2, %xmm4
 1937        pxor	%xmm1, %xmm6
 1938        movdqu	(%rsp), %xmm7
 1939        pshufd	$0x4e, %xmm15, %xmm1
 1940        pshufd	$0x4e, %xmm7, %xmm2
 1941        movdqa	%xmm7, %xmm3
 1942        movdqa	%xmm7, %xmm0
 1943        pclmulqdq	$0x11, %xmm15, %xmm3
 1944        pclmulqdq	$0x00, %xmm15, %xmm0
 1945        pxor	%xmm15, %xmm1
 1946        pxor	%xmm7, %xmm2
 1947        pclmulqdq	$0x00, %xmm2, %xmm1
 1948        pxor	%xmm0, %xmm1
 1949        pxor	%xmm3, %xmm1
 1950        movdqa	%xmm1, %xmm2
 1951        pxor	%xmm0, %xmm4
 1952        pxor	%xmm3, %xmm6
 1953        pslldq	$8, %xmm2
 1954        psrldq	$8, %xmm1
 1955        pxor	%xmm2, %xmm4
 1956        pxor	%xmm1, %xmm6
 1957        movdqa	%xmm4, %xmm0
 1958        movdqa	%xmm4, %xmm1
 1959        movdqa	%xmm4, %xmm2
 1960        pslld	$31, %xmm0
 1961        pslld	$30, %xmm1
 1962        pslld	$25, %xmm2
 1963        pxor	%xmm1, %xmm0
 1964        pxor	%xmm2, %xmm0
 1965        movdqa	%xmm0, %xmm1
 1966        psrldq	$4, %xmm1
 1967        pslldq	$12, %xmm0
 1968        pxor	%xmm0, %xmm4
 1969        movdqa	%xmm4, %xmm2
 1970        movdqa	%xmm4, %xmm3
 1971        movdqa	%xmm4, %xmm0
 1972        psrld	$0x01, %xmm2
 1973        psrld	$2, %xmm3
 1974        psrld	$7, %xmm0
 1975        pxor	%xmm3, %xmm2
 1976        pxor	%xmm0, %xmm2
 1977        pxor	%xmm1, %xmm2
 1978        pxor	%xmm4, %xmm2
 1979        pxor	%xmm2, %xmm6
 1980        movdqu	(%rsp), %xmm5
 1981L_AES_GCM_encrypt_aesni_done_128:
 1982        movl	%r9d, %edx
 1983        cmpl	%edx, %ebx
 1984        jge	L_AES_GCM_encrypt_aesni_done_enc
 1985        movl	%r9d, %r13d
 1986        andl	$0xfffffff0, %r13d
 1987        cmpl	%r13d, %ebx
 1988        jge	L_AES_GCM_encrypt_aesni_last_block_done
 1989        leaq	(%rdi,%rbx,1), %rcx
 1990        leaq	(%rsi,%rbx,1), %rdx
 1991        movdqu	128(%rsp), %xmm8
 1992        movdqa	%xmm8, %xmm9
 1993        pshufb	L_aes_gcm_bswap_epi64(%rip), %xmm8
 1994        paddd	L_aes_gcm_one(%rip), %xmm9
 1995        pxor	(%r15), %xmm8
 1996        movdqu	%xmm9, 128(%rsp)
 1997        aesenc	16(%r15), %xmm8
 1998        aesenc	32(%r15), %xmm8
 1999        aesenc	48(%r15), %xmm8
 2000        aesenc	64(%r15), %xmm8
 2001        aesenc	80(%r15), %xmm8
 2002        aesenc	96(%r15), %xmm8
 2003        aesenc	112(%r15), %xmm8
 2004        aesenc	128(%r15), %xmm8
 2005        aesenc	144(%r15), %xmm8
 2006        cmpl	$11, %r10d
 2007        movdqa	160(%r15), %xmm9
 2008        jl	L_AES_GCM_encrypt_aesni_aesenc_block_aesenc_avx_last
 2009        aesenc	%xmm9, %xmm8
 2010        aesenc	176(%r15), %xmm8
 2011        cmpl	$13, %r10d
 2012        movdqa	192(%r15), %xmm9
 2013        jl	L_AES_GCM_encrypt_aesni_aesenc_block_aesenc_avx_last
 2014        aesenc	%xmm9, %xmm8
 2015        aesenc	208(%r15), %xmm8
 2016        movdqa	224(%r15), %xmm9
 2017L_AES_GCM_encrypt_aesni_aesenc_block_aesenc_avx_last:
 2018        aesenclast	%xmm9, %xmm8
 2019        movdqu	(%rcx), %xmm9
 2020        pxor	%xmm9, %xmm8
 2021        movdqu	%xmm8, (%rdx)
 2022        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm8
 2023        pxor	%xmm8, %xmm6
 2024        addl	$16, %ebx
 2025        cmpl	%r13d, %ebx
 2026        jge	L_AES_GCM_encrypt_aesni_last_block_ghash
 2027L_AES_GCM_encrypt_aesni_last_block_start:
 2028        leaq	(%rdi,%rbx,1), %rcx
 2029        leaq	(%rsi,%rbx,1), %rdx
 2030        movdqu	128(%rsp), %xmm8
 2031        movdqa	%xmm8, %xmm9
 2032        pshufb	L_aes_gcm_bswap_epi64(%rip), %xmm8
 2033        paddd	L_aes_gcm_one(%rip), %xmm9
 2034        pxor	(%r15), %xmm8
 2035        movdqu	%xmm9, 128(%rsp)
 2036        movdqa	%xmm6, %xmm10
 2037        pclmulqdq	$16, %xmm5, %xmm10
 2038        aesenc	16(%r15), %xmm8
 2039        aesenc	32(%r15), %xmm8
 2040        movdqa	%xmm6, %xmm11
 2041        pclmulqdq	$0x01, %xmm5, %xmm11
 2042        aesenc	48(%r15), %xmm8
 2043        aesenc	64(%r15), %xmm8
 2044        movdqa	%xmm6, %xmm12
 2045        pclmulqdq	$0x00, %xmm5, %xmm12
 2046        aesenc	80(%r15), %xmm8
 2047        movdqa	%xmm6, %xmm1
 2048        pclmulqdq	$0x11, %xmm5, %xmm1
 2049        aesenc	96(%r15), %xmm8
 2050        pxor	%xmm11, %xmm10
 2051        movdqa	%xmm10, %xmm2
 2052        psrldq	$8, %xmm10
 2053        pslldq	$8, %xmm2
 2054        aesenc	112(%r15), %xmm8
 2055        movdqa	%xmm1, %xmm3
 2056        pxor	%xmm12, %xmm2
 2057        pxor	%xmm10, %xmm3
 2058        movdqa	L_aes_gcm_mod2_128(%rip), %xmm0
 2059        movdqa	%xmm2, %xmm11
 2060        pclmulqdq	$16, %xmm0, %xmm11
 2061        aesenc	128(%r15), %xmm8
 2062        pshufd	$0x4e, %xmm2, %xmm10
 2063        pxor	%xmm11, %xmm10
 2064        movdqa	%xmm10, %xmm11
 2065        pclmulqdq	$16, %xmm0, %xmm11
 2066        aesenc	144(%r15), %xmm8
 2067        pshufd	$0x4e, %xmm10, %xmm6
 2068        pxor	%xmm11, %xmm6
 2069        pxor	%xmm3, %xmm6
 2070        cmpl	$11, %r10d
 2071        movdqa	160(%r15), %xmm9
 2072        jl	L_AES_GCM_encrypt_aesni_aesenc_gfmul_last
 2073        aesenc	%xmm9, %xmm8
 2074        aesenc	176(%r15), %xmm8
 2075        cmpl	$13, %r10d
 2076        movdqa	192(%r15), %xmm9
 2077        jl	L_AES_GCM_encrypt_aesni_aesenc_gfmul_last
 2078        aesenc	%xmm9, %xmm8
 2079        aesenc	208(%r15), %xmm8
 2080        movdqa	224(%r15), %xmm9
 2081L_AES_GCM_encrypt_aesni_aesenc_gfmul_last:
 2082        aesenclast	%xmm9, %xmm8
 2083        movdqu	(%rcx), %xmm9
 2084        pxor	%xmm9, %xmm8
 2085        movdqu	%xmm8, (%rdx)
 2086        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm8
 2087        pxor	%xmm8, %xmm6
 2088        addl	$16, %ebx
 2089        cmpl	%r13d, %ebx
 2090        jl	L_AES_GCM_encrypt_aesni_last_block_start
 2091L_AES_GCM_encrypt_aesni_last_block_ghash:
 2092        pshufd	$0x4e, %xmm5, %xmm9
 2093        pshufd	$0x4e, %xmm6, %xmm10
 2094        movdqa	%xmm6, %xmm11
 2095        movdqa	%xmm6, %xmm8
 2096        pclmulqdq	$0x11, %xmm5, %xmm11
 2097        pclmulqdq	$0x00, %xmm5, %xmm8
 2098        pxor	%xmm5, %xmm9
 2099        pxor	%xmm6, %xmm10
 2100        pclmulqdq	$0x00, %xmm10, %xmm9
 2101        pxor	%xmm8, %xmm9
 2102        pxor	%xmm11, %xmm9
 2103        movdqa	%xmm9, %xmm10
 2104        movdqa	%xmm11, %xmm6
 2105        pslldq	$8, %xmm10
 2106        psrldq	$8, %xmm9
 2107        pxor	%xmm10, %xmm8
 2108        pxor	%xmm9, %xmm6
 2109        movdqa	%xmm8, %xmm12
 2110        movdqa	%xmm8, %xmm13
 2111        movdqa	%xmm8, %xmm14
 2112        pslld	$31, %xmm12
 2113        pslld	$30, %xmm13
 2114        pslld	$25, %xmm14
 2115        pxor	%xmm13, %xmm12
 2116        pxor	%xmm14, %xmm12
 2117        movdqa	%xmm12, %xmm13
 2118        psrldq	$4, %xmm13
 2119        pslldq	$12, %xmm12
 2120        pxor	%xmm12, %xmm8
 2121        movdqa	%xmm8, %xmm14
 2122        movdqa	%xmm8, %xmm10
 2123        movdqa	%xmm8, %xmm9
 2124        psrld	$0x01, %xmm14
 2125        psrld	$2, %xmm10
 2126        psrld	$7, %xmm9
 2127        pxor	%xmm10, %xmm14
 2128        pxor	%xmm9, %xmm14
 2129        pxor	%xmm13, %xmm14
 2130        pxor	%xmm8, %xmm14
 2131        pxor	%xmm14, %xmm6
 2132L_AES_GCM_encrypt_aesni_last_block_done:
 2133        movl	%r9d, %ecx
 2134        movl	%ecx, %edx
 2135        andl	$15, %ecx
 2136        jz	L_AES_GCM_encrypt_aesni_aesenc_last15_enc_avx_done
 2137        movdqu	128(%rsp), %xmm4
 2138        pshufb	L_aes_gcm_bswap_epi64(%rip), %xmm4
 2139        pxor	(%r15), %xmm4
 2140        aesenc	16(%r15), %xmm4
 2141        aesenc	32(%r15), %xmm4
 2142        aesenc	48(%r15), %xmm4
 2143        aesenc	64(%r15), %xmm4
 2144        aesenc	80(%r15), %xmm4
 2145        aesenc	96(%r15), %xmm4
 2146        aesenc	112(%r15), %xmm4
 2147        aesenc	128(%r15), %xmm4
 2148        aesenc	144(%r15), %xmm4
 2149        cmpl	$11, %r10d
 2150        movdqa	160(%r15), %xmm9
 2151        jl	L_AES_GCM_encrypt_aesni_aesenc_last15_enc_avx_aesenc_avx_last
 2152        aesenc	%xmm9, %xmm4
 2153        aesenc	176(%r15), %xmm4
 2154        cmpl	$13, %r10d
 2155        movdqa	192(%r15), %xmm9
 2156        jl	L_AES_GCM_encrypt_aesni_aesenc_last15_enc_avx_aesenc_avx_last
 2157        aesenc	%xmm9, %xmm4
 2158        aesenc	208(%r15), %xmm4
 2159        movdqa	224(%r15), %xmm9
 2160L_AES_GCM_encrypt_aesni_aesenc_last15_enc_avx_aesenc_avx_last:
 2161        aesenclast	%xmm9, %xmm4
 2162        subq	$16, %rsp
 2163        xorl	%ecx, %ecx
 2164        movdqu	%xmm4, (%rsp)
 2165L_AES_GCM_encrypt_aesni_aesenc_last15_enc_avx_loop:
 2166        movzbl	(%rdi,%rbx,1), %r13d
 2167        xorb	(%rsp,%rcx,1), %r13b
 2168        movb	%r13b, (%rsi,%rbx,1)
 2169        movb	%r13b, (%rsp,%rcx,1)
 2170        incl	%ebx
 2171        incl	%ecx
 2172        cmpl	%edx, %ebx
 2173        jl	L_AES_GCM_encrypt_aesni_aesenc_last15_enc_avx_loop
 2174        xorq	%r13, %r13
 2175        cmpl	$16, %ecx
 2176        je	L_AES_GCM_encrypt_aesni_aesenc_last15_enc_avx_finish_enc
 2177L_AES_GCM_encrypt_aesni_aesenc_last15_enc_avx_byte_loop:
 2178        movb	%r13b, (%rsp,%rcx,1)
 2179        incl	%ecx
 2180        cmpl	$16, %ecx
 2181        jl	L_AES_GCM_encrypt_aesni_aesenc_last15_enc_avx_byte_loop
 2182L_AES_GCM_encrypt_aesni_aesenc_last15_enc_avx_finish_enc:
 2183        movdqu	(%rsp), %xmm4
 2184        addq	$16, %rsp
 2185        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm4
 2186        pxor	%xmm4, %xmm6
 2187        pshufd	$0x4e, %xmm5, %xmm9
 2188        pshufd	$0x4e, %xmm6, %xmm10
 2189        movdqa	%xmm6, %xmm11
 2190        movdqa	%xmm6, %xmm8
 2191        pclmulqdq	$0x11, %xmm5, %xmm11
 2192        pclmulqdq	$0x00, %xmm5, %xmm8
 2193        pxor	%xmm5, %xmm9
 2194        pxor	%xmm6, %xmm10
 2195        pclmulqdq	$0x00, %xmm10, %xmm9
 2196        pxor	%xmm8, %xmm9
 2197        pxor	%xmm11, %xmm9
 2198        movdqa	%xmm9, %xmm10
 2199        movdqa	%xmm11, %xmm6
 2200        pslldq	$8, %xmm10
 2201        psrldq	$8, %xmm9
 2202        pxor	%xmm10, %xmm8
 2203        pxor	%xmm9, %xmm6
 2204        movdqa	%xmm8, %xmm12
 2205        movdqa	%xmm8, %xmm13
 2206        movdqa	%xmm8, %xmm14
 2207        pslld	$31, %xmm12
 2208        pslld	$30, %xmm13
 2209        pslld	$25, %xmm14
 2210        pxor	%xmm13, %xmm12
 2211        pxor	%xmm14, %xmm12
 2212        movdqa	%xmm12, %xmm13
 2213        psrldq	$4, %xmm13
 2214        pslldq	$12, %xmm12
 2215        pxor	%xmm12, %xmm8
 2216        movdqa	%xmm8, %xmm14
 2217        movdqa	%xmm8, %xmm10
 2218        movdqa	%xmm8, %xmm9
 2219        psrld	$0x01, %xmm14
 2220        psrld	$2, %xmm10
 2221        psrld	$7, %xmm9
 2222        pxor	%xmm10, %xmm14
 2223        pxor	%xmm9, %xmm14
 2224        pxor	%xmm13, %xmm14
 2225        pxor	%xmm8, %xmm14
 2226        pxor	%xmm14, %xmm6
 2227L_AES_GCM_encrypt_aesni_aesenc_last15_enc_avx_done:
 2228L_AES_GCM_encrypt_aesni_done_enc:
 2229        movl	%r9d, %edx
 2230        movl	%r11d, %ecx
 2231        shlq	$3, %rdx
 2232        shlq	$3, %rcx
 2233        pinsrq	$0x00, %rdx, %xmm0
 2234        pinsrq	$0x01, %rcx, %xmm0
 2235        pxor	%xmm0, %xmm6
 2236        pshufd	$0x4e, %xmm5, %xmm9
 2237        pshufd	$0x4e, %xmm6, %xmm10
 2238        movdqa	%xmm6, %xmm11
 2239        movdqa	%xmm6, %xmm8
 2240        pclmulqdq	$0x11, %xmm5, %xmm11
 2241        pclmulqdq	$0x00, %xmm5, %xmm8
 2242        pxor	%xmm5, %xmm9
 2243        pxor	%xmm6, %xmm10
 2244        pclmulqdq	$0x00, %xmm10, %xmm9
 2245        pxor	%xmm8, %xmm9
 2246        pxor	%xmm11, %xmm9
 2247        movdqa	%xmm9, %xmm10
 2248        movdqa	%xmm11, %xmm6
 2249        pslldq	$8, %xmm10
 2250        psrldq	$8, %xmm9
 2251        pxor	%xmm10, %xmm8
 2252        pxor	%xmm9, %xmm6
 2253        movdqa	%xmm8, %xmm12
 2254        movdqa	%xmm8, %xmm13
 2255        movdqa	%xmm8, %xmm14
 2256        pslld	$31, %xmm12
 2257        pslld	$30, %xmm13
 2258        pslld	$25, %xmm14
 2259        pxor	%xmm13, %xmm12
 2260        pxor	%xmm14, %xmm12
 2261        movdqa	%xmm12, %xmm13
 2262        psrldq	$4, %xmm13
 2263        pslldq	$12, %xmm12
 2264        pxor	%xmm12, %xmm8
 2265        movdqa	%xmm8, %xmm14
 2266        movdqa	%xmm8, %xmm10
 2267        movdqa	%xmm8, %xmm9
 2268        psrld	$0x01, %xmm14
 2269        psrld	$2, %xmm10
 2270        psrld	$7, %xmm9
 2271        pxor	%xmm10, %xmm14
 2272        pxor	%xmm9, %xmm14
 2273        pxor	%xmm13, %xmm14
 2274        pxor	%xmm8, %xmm14
 2275        pxor	%xmm14, %xmm6
 2276        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm6
 2277        movdqu	144(%rsp), %xmm0
 2278        pxor	%xmm6, %xmm0
 2279        cmpl	$16, %r14d
 2280        je	L_AES_GCM_encrypt_aesni_store_tag_16
 2281        xorq	%rcx, %rcx
 2282        movdqu	%xmm0, (%rsp)
 2283L_AES_GCM_encrypt_aesni_store_tag_loop:
 2284        movzbl	(%rsp,%rcx,1), %r13d
 2285        movb	%r13b, (%r8,%rcx,1)
 2286        incl	%ecx
 2287        cmpl	%r14d, %ecx
 2288        jne	L_AES_GCM_encrypt_aesni_store_tag_loop
 2289        jmp	L_AES_GCM_encrypt_aesni_store_tag_done
 2290L_AES_GCM_encrypt_aesni_store_tag_16:
 2291        movdqu	%xmm0, (%r8)
 2292L_AES_GCM_encrypt_aesni_store_tag_done:
 2293        addq	$0xa0, %rsp
 2294        popq	%r15
 2295        popq	%r14
 2296        popq	%rbx
 2297        popq	%r12
 2298        popq	%r13
 2299        repz retq
 2300#ifndef __APPLE__
 2301.size	AES_GCM_encrypt_aesni,.-AES_GCM_encrypt_aesni
 2302#endif /* __APPLE__ */
 2303#ifndef __APPLE__
 2304.text
 2305.globl	AES_GCM_decrypt_aesni
 2306.type	AES_GCM_decrypt_aesni,@function
 2307.align	16
 2308AES_GCM_decrypt_aesni:
 2309#else
 2310.section	__TEXT,__text
 2311.globl	_AES_GCM_decrypt_aesni
 2312.p2align	4
 2313_AES_GCM_decrypt_aesni:
 2314#endif /* __APPLE__ */
 2315        pushq	%r13
 2316        pushq	%r12
 2317        pushq	%rbx
 2318        pushq	%r14
 2319        pushq	%r15
 2320        pushq	%rbp
 2321        movq	%rdx, %r12
 2322        movq	%rcx, %rax
 2323        movl	56(%rsp), %r11d
 2324        movl	64(%rsp), %ebx
 2325        movl	72(%rsp), %r14d
 2326        movq	80(%rsp), %r15
 2327        movl	88(%rsp), %r10d
 2328        movq	96(%rsp), %rbp
 2329        subq	$0xa8, %rsp
 2330        pxor	%xmm4, %xmm4
 2331        pxor	%xmm6, %xmm6
 2332        cmpl	$12, %ebx
 2333        movl	%ebx, %edx
 2334        jne	L_AES_GCM_decrypt_aesni_iv_not_12
 2335        # # Calculate values when IV is 12 bytes
 2336        # Set counter based on IV
 2337        movl	$0x1000000, %ecx
 2338        pinsrq	$0x00, (%rax), %xmm4
 2339        pinsrd	$2, 8(%rax), %xmm4
 2340        pinsrd	$3, %ecx, %xmm4
 2341        # H = Encrypt X(=0) and T = Encrypt counter
 2342        movdqa	%xmm4, %xmm1
 2343        movdqa	(%r15), %xmm5
 2344        pxor	%xmm5, %xmm1
 2345        movdqa	16(%r15), %xmm7
 2346        aesenc	%xmm7, %xmm5
 2347        aesenc	%xmm7, %xmm1
 2348        movdqa	32(%r15), %xmm7
 2349        aesenc	%xmm7, %xmm5
 2350        aesenc	%xmm7, %xmm1
 2351        movdqa	48(%r15), %xmm7
 2352        aesenc	%xmm7, %xmm5
 2353        aesenc	%xmm7, %xmm1
 2354        movdqa	64(%r15), %xmm7
 2355        aesenc	%xmm7, %xmm5
 2356        aesenc	%xmm7, %xmm1
 2357        movdqa	80(%r15), %xmm7
 2358        aesenc	%xmm7, %xmm5
 2359        aesenc	%xmm7, %xmm1
 2360        movdqa	96(%r15), %xmm7
 2361        aesenc	%xmm7, %xmm5
 2362        aesenc	%xmm7, %xmm1
 2363        movdqa	112(%r15), %xmm7
 2364        aesenc	%xmm7, %xmm5
 2365        aesenc	%xmm7, %xmm1
 2366        movdqa	128(%r15), %xmm7
 2367        aesenc	%xmm7, %xmm5
 2368        aesenc	%xmm7, %xmm1
 2369        movdqa	144(%r15), %xmm7
 2370        aesenc	%xmm7, %xmm5
 2371        aesenc	%xmm7, %xmm1
 2372        cmpl	$11, %r10d
 2373        movdqa	160(%r15), %xmm7
 2374        jl	L_AES_GCM_decrypt_aesni_calc_iv_12_last
 2375        aesenc	%xmm7, %xmm5
 2376        aesenc	%xmm7, %xmm1
 2377        movdqa	176(%r15), %xmm7
 2378        aesenc	%xmm7, %xmm5
 2379        aesenc	%xmm7, %xmm1
 2380        cmpl	$13, %r10d
 2381        movdqa	192(%r15), %xmm7
 2382        jl	L_AES_GCM_decrypt_aesni_calc_iv_12_last
 2383        aesenc	%xmm7, %xmm5
 2384        aesenc	%xmm7, %xmm1
 2385        movdqa	208(%r15), %xmm7
 2386        aesenc	%xmm7, %xmm5
 2387        aesenc	%xmm7, %xmm1
 2388        movdqa	224(%r15), %xmm7
 2389L_AES_GCM_decrypt_aesni_calc_iv_12_last:
 2390        aesenclast	%xmm7, %xmm5
 2391        aesenclast	%xmm7, %xmm1
 2392        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm5
 2393        movdqu	%xmm1, 144(%rsp)
 2394        jmp	L_AES_GCM_decrypt_aesni_iv_done
 2395L_AES_GCM_decrypt_aesni_iv_not_12:
 2396        # Calculate values when IV is not 12 bytes
 2397        # H = Encrypt X(=0)
 2398        movdqa	(%r15), %xmm5
 2399        aesenc	16(%r15), %xmm5
 2400        aesenc	32(%r15), %xmm5
 2401        aesenc	48(%r15), %xmm5
 2402        aesenc	64(%r15), %xmm5
 2403        aesenc	80(%r15), %xmm5
 2404        aesenc	96(%r15), %xmm5
 2405        aesenc	112(%r15), %xmm5
 2406        aesenc	128(%r15), %xmm5
 2407        aesenc	144(%r15), %xmm5
 2408        cmpl	$11, %r10d
 2409        movdqa	160(%r15), %xmm9
 2410        jl	L_AES_GCM_decrypt_aesni_calc_iv_1_aesenc_avx_last
 2411        aesenc	%xmm9, %xmm5
 2412        aesenc	176(%r15), %xmm5
 2413        cmpl	$13, %r10d
 2414        movdqa	192(%r15), %xmm9
 2415        jl	L_AES_GCM_decrypt_aesni_calc_iv_1_aesenc_avx_last
 2416        aesenc	%xmm9, %xmm5
 2417        aesenc	208(%r15), %xmm5
 2418        movdqa	224(%r15), %xmm9
 2419L_AES_GCM_decrypt_aesni_calc_iv_1_aesenc_avx_last:
 2420        aesenclast	%xmm9, %xmm5
 2421        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm5
 2422        # Calc counter
 2423        # Initialization vector
 2424        cmpl	$0x00, %edx
 2425        movq	$0x00, %rcx
 2426        je	L_AES_GCM_decrypt_aesni_calc_iv_done
 2427        cmpl	$16, %edx
 2428        jl	L_AES_GCM_decrypt_aesni_calc_iv_lt16
 2429        andl	$0xfffffff0, %edx
 2430L_AES_GCM_decrypt_aesni_calc_iv_16_loop:
 2431        movdqu	(%rax,%rcx,1), %xmm8
 2432        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm8
 2433        pxor	%xmm8, %xmm4
 2434        pshufd	$0x4e, %xmm4, %xmm1
 2435        pshufd	$0x4e, %xmm5, %xmm2
 2436        movdqa	%xmm5, %xmm3
 2437        movdqa	%xmm5, %xmm0
 2438        pclmulqdq	$0x11, %xmm4, %xmm3
 2439        pclmulqdq	$0x00, %xmm4, %xmm0
 2440        pxor	%xmm4, %xmm1
 2441        pxor	%xmm5, %xmm2
 2442        pclmulqdq	$0x00, %xmm2, %xmm1
 2443        pxor	%xmm0, %xmm1
 2444        pxor	%xmm3, %xmm1
 2445        movdqa	%xmm1, %xmm2
 2446        movdqa	%xmm0, %xmm7
 2447        movdqa	%xmm3, %xmm4
 2448        pslldq	$8, %xmm2
 2449        psrldq	$8, %xmm1
 2450        pxor	%xmm2, %xmm7
 2451        pxor	%xmm1, %xmm4
 2452        movdqa	%xmm7, %xmm0
 2453        movdqa	%xmm4, %xmm1
 2454        psrld	$31, %xmm0
 2455        psrld	$31, %xmm1
 2456        pslld	$0x01, %xmm7
 2457        pslld	$0x01, %xmm4
 2458        movdqa	%xmm0, %xmm2
 2459        pslldq	$4, %xmm0
 2460        psrldq	$12, %xmm2
 2461        pslldq	$4, %xmm1
 2462        por	%xmm2, %xmm4
 2463        por	%xmm0, %xmm7
 2464        por	%xmm1, %xmm4
 2465        movdqa	%xmm7, %xmm0
 2466        movdqa	%xmm7, %xmm1
 2467        movdqa	%xmm7, %xmm2
 2468        pslld	$31, %xmm0
 2469        pslld	$30, %xmm1
 2470        pslld	$25, %xmm2
 2471        pxor	%xmm1, %xmm0
 2472        pxor	%xmm2, %xmm0
 2473        movdqa	%xmm0, %xmm1
 2474        psrldq	$4, %xmm1
 2475        pslldq	$12, %xmm0
 2476        pxor	%xmm0, %xmm7
 2477        movdqa	%xmm7, %xmm2
 2478        movdqa	%xmm7, %xmm3
 2479        movdqa	%xmm7, %xmm0
 2480        psrld	$0x01, %xmm2
 2481        psrld	$2, %xmm3
 2482        psrld	$7, %xmm0
 2483        pxor	%xmm3, %xmm2
 2484        pxor	%xmm0, %xmm2
 2485        pxor	%xmm1, %xmm2
 2486        pxor	%xmm7, %xmm2
 2487        pxor	%xmm2, %xmm4
 2488        addl	$16, %ecx
 2489        cmpl	%edx, %ecx
 2490        jl	L_AES_GCM_decrypt_aesni_calc_iv_16_loop
 2491        movl	%ebx, %edx
 2492        cmpl	%edx, %ecx
 2493        je	L_AES_GCM_decrypt_aesni_calc_iv_done
 2494L_AES_GCM_decrypt_aesni_calc_iv_lt16:
 2495        subq	$16, %rsp
 2496        pxor	%xmm8, %xmm8
 2497        xorl	%ebx, %ebx
 2498        movdqu	%xmm8, (%rsp)
 2499L_AES_GCM_decrypt_aesni_calc_iv_loop:
 2500        movzbl	(%rax,%rcx,1), %r13d
 2501        movb	%r13b, (%rsp,%rbx,1)
 2502        incl	%ecx
 2503        incl	%ebx
 2504        cmpl	%edx, %ecx
 2505        jl	L_AES_GCM_decrypt_aesni_calc_iv_loop
 2506        movdqu	(%rsp), %xmm8
 2507        addq	$16, %rsp
 2508        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm8
 2509        pxor	%xmm8, %xmm4
 2510        pshufd	$0x4e, %xmm4, %xmm1
 2511        pshufd	$0x4e, %xmm5, %xmm2
 2512        movdqa	%xmm5, %xmm3
 2513        movdqa	%xmm5, %xmm0
 2514        pclmulqdq	$0x11, %xmm4, %xmm3
 2515        pclmulqdq	$0x00, %xmm4, %xmm0
 2516        pxor	%xmm4, %xmm1
 2517        pxor	%xmm5, %xmm2
 2518        pclmulqdq	$0x00, %xmm2, %xmm1
 2519        pxor	%xmm0, %xmm1
 2520        pxor	%xmm3, %xmm1
 2521        movdqa	%xmm1, %xmm2
 2522        movdqa	%xmm0, %xmm7
 2523        movdqa	%xmm3, %xmm4
 2524        pslldq	$8, %xmm2
 2525        psrldq	$8, %xmm1
 2526        pxor	%xmm2, %xmm7
 2527        pxor	%xmm1, %xmm4
 2528        movdqa	%xmm7, %xmm0
 2529        movdqa	%xmm4, %xmm1
 2530        psrld	$31, %xmm0
 2531        psrld	$31, %xmm1
 2532        pslld	$0x01, %xmm7
 2533        pslld	$0x01, %xmm4
 2534        movdqa	%xmm0, %xmm2
 2535        pslldq	$4, %xmm0
 2536        psrldq	$12, %xmm2
 2537        pslldq	$4, %xmm1
 2538        por	%xmm2, %xmm4
 2539        por	%xmm0, %xmm7
 2540        por	%xmm1, %xmm4
 2541        movdqa	%xmm7, %xmm0
 2542        movdqa	%xmm7, %xmm1
 2543        movdqa	%xmm7, %xmm2
 2544        pslld	$31, %xmm0
 2545        pslld	$30, %xmm1
 2546        pslld	$25, %xmm2
 2547        pxor	%xmm1, %xmm0
 2548        pxor	%xmm2, %xmm0
 2549        movdqa	%xmm0, %xmm1
 2550        psrldq	$4, %xmm1
 2551        pslldq	$12, %xmm0
 2552        pxor	%xmm0, %xmm7
 2553        movdqa	%xmm7, %xmm2
 2554        movdqa	%xmm7, %xmm3
 2555        movdqa	%xmm7, %xmm0
 2556        psrld	$0x01, %xmm2
 2557        psrld	$2, %xmm3
 2558        psrld	$7, %xmm0
 2559        pxor	%xmm3, %xmm2
 2560        pxor	%xmm0, %xmm2
 2561        pxor	%xmm1, %xmm2
 2562        pxor	%xmm7, %xmm2
 2563        pxor	%xmm2, %xmm4
 2564L_AES_GCM_decrypt_aesni_calc_iv_done:
 2565        # T = Encrypt counter
 2566        pxor	%xmm0, %xmm0
 2567        shll	$3, %edx
 2568        pinsrq	$0x00, %rdx, %xmm0
 2569        pxor	%xmm0, %xmm4
 2570        pshufd	$0x4e, %xmm4, %xmm1
 2571        pshufd	$0x4e, %xmm5, %xmm2
 2572        movdqa	%xmm5, %xmm3
 2573        movdqa	%xmm5, %xmm0
 2574        pclmulqdq	$0x11, %xmm4, %xmm3
 2575        pclmulqdq	$0x00, %xmm4, %xmm0
 2576        pxor	%xmm4, %xmm1
 2577        pxor	%xmm5, %xmm2
 2578        pclmulqdq	$0x00, %xmm2, %xmm1
 2579        pxor	%xmm0, %xmm1
 2580        pxor	%xmm3, %xmm1
 2581        movdqa	%xmm1, %xmm2
 2582        movdqa	%xmm0, %xmm7
 2583        movdqa	%xmm3, %xmm4
 2584        pslldq	$8, %xmm2
 2585        psrldq	$8, %xmm1
 2586        pxor	%xmm2, %xmm7
 2587        pxor	%xmm1, %xmm4
 2588        movdqa	%xmm7, %xmm0
 2589        movdqa	%xmm4, %xmm1
 2590        psrld	$31, %xmm0
 2591        psrld	$31, %xmm1
 2592        pslld	$0x01, %xmm7
 2593        pslld	$0x01, %xmm4
 2594        movdqa	%xmm0, %xmm2
 2595        pslldq	$4, %xmm0
 2596        psrldq	$12, %xmm2
 2597        pslldq	$4, %xmm1
 2598        por	%xmm2, %xmm4
 2599        por	%xmm0, %xmm7
 2600        por	%xmm1, %xmm4
 2601        movdqa	%xmm7, %xmm0
 2602        movdqa	%xmm7, %xmm1
 2603        movdqa	%xmm7, %xmm2
 2604        pslld	$31, %xmm0
 2605        pslld	$30, %xmm1
 2606        pslld	$25, %xmm2
 2607        pxor	%xmm1, %xmm0
 2608        pxor	%xmm2, %xmm0
 2609        movdqa	%xmm0, %xmm1
 2610        psrldq	$4, %xmm1
 2611        pslldq	$12, %xmm0
 2612        pxor	%xmm0, %xmm7
 2613        movdqa	%xmm7, %xmm2
 2614        movdqa	%xmm7, %xmm3
 2615        movdqa	%xmm7, %xmm0
 2616        psrld	$0x01, %xmm2
 2617        psrld	$2, %xmm3
 2618        psrld	$7, %xmm0
 2619        pxor	%xmm3, %xmm2
 2620        pxor	%xmm0, %xmm2
 2621        pxor	%xmm1, %xmm2
 2622        pxor	%xmm7, %xmm2
 2623        pxor	%xmm2, %xmm4
 2624        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm4
 2625        #   Encrypt counter
 2626        movdqa	(%r15), %xmm8
 2627        pxor	%xmm4, %xmm8
 2628        aesenc	16(%r15), %xmm8
 2629        aesenc	32(%r15), %xmm8
 2630        aesenc	48(%r15), %xmm8
 2631        aesenc	64(%r15), %xmm8
 2632        aesenc	80(%r15), %xmm8
 2633        aesenc	96(%r15), %xmm8
 2634        aesenc	112(%r15), %xmm8
 2635        aesenc	128(%r15), %xmm8
 2636        aesenc	144(%r15), %xmm8
 2637        cmpl	$11, %r10d
 2638        movdqa	160(%r15), %xmm9
 2639        jl	L_AES_GCM_decrypt_aesni_calc_iv_2_aesenc_avx_last
 2640        aesenc	%xmm9, %xmm8
 2641        aesenc	176(%r15), %xmm8
 2642        cmpl	$13, %r10d
 2643        movdqa	192(%r15), %xmm9
 2644        jl	L_AES_GCM_decrypt_aesni_calc_iv_2_aesenc_avx_last
 2645        aesenc	%xmm9, %xmm8
 2646        aesenc	208(%r15), %xmm8
 2647        movdqa	224(%r15), %xmm9
 2648L_AES_GCM_decrypt_aesni_calc_iv_2_aesenc_avx_last:
 2649        aesenclast	%xmm9, %xmm8
 2650        movdqu	%xmm8, 144(%rsp)
 2651L_AES_GCM_decrypt_aesni_iv_done:
 2652        # Additional authentication data
 2653        movl	%r11d, %edx
 2654        cmpl	$0x00, %edx
 2655        je	L_AES_GCM_decrypt_aesni_calc_aad_done
 2656        xorl	%ecx, %ecx
 2657        cmpl	$16, %edx
 2658        jl	L_AES_GCM_decrypt_aesni_calc_aad_lt16
 2659        andl	$0xfffffff0, %edx
 2660L_AES_GCM_decrypt_aesni_calc_aad_16_loop:
 2661        movdqu	(%r12,%rcx,1), %xmm8
 2662        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm8
 2663        pxor	%xmm8, %xmm6
 2664        pshufd	$0x4e, %xmm6, %xmm1
 2665        pshufd	$0x4e, %xmm5, %xmm2
 2666        movdqa	%xmm5, %xmm3
 2667        movdqa	%xmm5, %xmm0
 2668        pclmulqdq	$0x11, %xmm6, %xmm3
 2669        pclmulqdq	$0x00, %xmm6, %xmm0
 2670        pxor	%xmm6, %xmm1
 2671        pxor	%xmm5, %xmm2
 2672        pclmulqdq	$0x00, %xmm2, %xmm1
 2673        pxor	%xmm0, %xmm1
 2674        pxor	%xmm3, %xmm1
 2675        movdqa	%xmm1, %xmm2
 2676        movdqa	%xmm0, %xmm7
 2677        movdqa	%xmm3, %xmm6
 2678        pslldq	$8, %xmm2
 2679        psrldq	$8, %xmm1
 2680        pxor	%xmm2, %xmm7
 2681        pxor	%xmm1, %xmm6
 2682        movdqa	%xmm7, %xmm0
 2683        movdqa	%xmm6, %xmm1
 2684        psrld	$31, %xmm0
 2685        psrld	$31, %xmm1
 2686        pslld	$0x01, %xmm7
 2687        pslld	$0x01, %xmm6
 2688        movdqa	%xmm0, %xmm2
 2689        pslldq	$4, %xmm0
 2690        psrldq	$12, %xmm2
 2691        pslldq	$4, %xmm1
 2692        por	%xmm2, %xmm6
 2693        por	%xmm0, %xmm7
 2694        por	%xmm1, %xmm6
 2695        movdqa	%xmm7, %xmm0
 2696        movdqa	%xmm7, %xmm1
 2697        movdqa	%xmm7, %xmm2
 2698        pslld	$31, %xmm0
 2699        pslld	$30, %xmm1
 2700        pslld	$25, %xmm2
 2701        pxor	%xmm1, %xmm0
 2702        pxor	%xmm2, %xmm0
 2703        movdqa	%xmm0, %xmm1
 2704        psrldq	$4, %xmm1
 2705        pslldq	$12, %xmm0
 2706        pxor	%xmm0, %xmm7
 2707        movdqa	%xmm7, %xmm2
 2708        movdqa	%xmm7, %xmm3
 2709        movdqa	%xmm7, %xmm0
 2710        psrld	$0x01, %xmm2
 2711        psrld	$2, %xmm3
 2712        psrld	$7, %xmm0
 2713        pxor	%xmm3, %xmm2
 2714        pxor	%xmm0, %xmm2
 2715        pxor	%xmm1, %xmm2
 2716        pxor	%xmm7, %xmm2
 2717        pxor	%xmm2, %xmm6
 2718        addl	$16, %ecx
 2719        cmpl	%edx, %ecx
 2720        jl	L_AES_GCM_decrypt_aesni_calc_aad_16_loop
 2721        movl	%r11d, %edx
 2722        cmpl	%edx, %ecx
 2723        je	L_AES_GCM_decrypt_aesni_calc_aad_done
 2724L_AES_GCM_decrypt_aesni_calc_aad_lt16:
 2725        subq	$16, %rsp
 2726        pxor	%xmm8, %xmm8
 2727        xorl	%ebx, %ebx
 2728        movdqu	%xmm8, (%rsp)
 2729L_AES_GCM_decrypt_aesni_calc_aad_loop:
 2730        movzbl	(%r12,%rcx,1), %r13d
 2731        movb	%r13b, (%rsp,%rbx,1)
 2732        incl	%ecx
 2733        incl	%ebx
 2734        cmpl	%edx, %ecx
 2735        jl	L_AES_GCM_decrypt_aesni_calc_aad_loop
 2736        movdqu	(%rsp), %xmm8
 2737        addq	$16, %rsp
 2738        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm8
 2739        pxor	%xmm8, %xmm6
 2740        pshufd	$0x4e, %xmm6, %xmm1
 2741        pshufd	$0x4e, %xmm5, %xmm2
 2742        movdqa	%xmm5, %xmm3
 2743        movdqa	%xmm5, %xmm0
 2744        pclmulqdq	$0x11, %xmm6, %xmm3
 2745        pclmulqdq	$0x00, %xmm6, %xmm0
 2746        pxor	%xmm6, %xmm1
 2747        pxor	%xmm5, %xmm2
 2748        pclmulqdq	$0x00, %xmm2, %xmm1
 2749        pxor	%xmm0, %xmm1
 2750        pxor	%xmm3, %xmm1
 2751        movdqa	%xmm1, %xmm2
 2752        movdqa	%xmm0, %xmm7
 2753        movdqa	%xmm3, %xmm6
 2754        pslldq	$8, %xmm2
 2755        psrldq	$8, %xmm1
 2756        pxor	%xmm2, %xmm7
 2757        pxor	%xmm1, %xmm6
 2758        movdqa	%xmm7, %xmm0
 2759        movdqa	%xmm6, %xmm1
 2760        psrld	$31, %xmm0
 2761        psrld	$31, %xmm1
 2762        pslld	$0x01, %xmm7
 2763        pslld	$0x01, %xmm6
 2764        movdqa	%xmm0, %xmm2
 2765        pslldq	$4, %xmm0
 2766        psrldq	$12, %xmm2
 2767        pslldq	$4, %xmm1
 2768        por	%xmm2, %xmm6
 2769        por	%xmm0, %xmm7
 2770        por	%xmm1, %xmm6
 2771        movdqa	%xmm7, %xmm0
 2772        movdqa	%xmm7, %xmm1
 2773        movdqa	%xmm7, %xmm2
 2774        pslld	$31, %xmm0
 2775        pslld	$30, %xmm1
 2776        pslld	$25, %xmm2
 2777        pxor	%xmm1, %xmm0
 2778        pxor	%xmm2, %xmm0
 2779        movdqa	%xmm0, %xmm1
 2780        psrldq	$4, %xmm1
 2781        pslldq	$12, %xmm0
 2782        pxor	%xmm0, %xmm7
 2783        movdqa	%xmm7, %xmm2
 2784        movdqa	%xmm7, %xmm3
 2785        movdqa	%xmm7, %xmm0
 2786        psrld	$0x01, %xmm2
 2787        psrld	$2, %xmm3
 2788        psrld	$7, %xmm0
 2789        pxor	%xmm3, %xmm2
 2790        pxor	%xmm0, %xmm2
 2791        pxor	%xmm1, %xmm2
 2792        pxor	%xmm7, %xmm2
 2793        pxor	%xmm2, %xmm6
 2794L_AES_GCM_decrypt_aesni_calc_aad_done:
 2795        # Calculate counter and H
 2796        pshufb	L_aes_gcm_bswap_epi64(%rip), %xmm4
 2797        movdqa	%xmm5, %xmm9
 2798        paddd	L_aes_gcm_one(%rip), %xmm4
 2799        movdqa	%xmm5, %xmm8
 2800        movdqu	%xmm4, 128(%rsp)
 2801        psrlq	$63, %xmm9
 2802        psllq	$0x01, %xmm8
 2803        pslldq	$8, %xmm9
 2804        por	%xmm9, %xmm8
 2805        pshufd	$0xff, %xmm5, %xmm5
 2806        psrad	$31, %xmm5
 2807        pand	L_aes_gcm_mod2_128(%rip), %xmm5
 2808        pxor	%xmm8, %xmm5
 2809        xorl	%ebx, %ebx
 2810        cmpl	$0x80, %r9d
 2811        movl	%r9d, %r13d
 2812        jl	L_AES_GCM_decrypt_aesni_done_128
 2813        andl	$0xffffff80, %r13d
 2814        movdqa	%xmm6, %xmm2
 2815        # H ^ 1
 2816        movdqu	%xmm5, (%rsp)
 2817        # H ^ 2
 2818        pshufd	$0x4e, %xmm5, %xmm9
 2819        pshufd	$0x4e, %xmm5, %xmm10
 2820        movdqa	%xmm5, %xmm11
 2821        movdqa	%xmm5, %xmm8
 2822        pclmulqdq	$0x11, %xmm5, %xmm11
 2823        pclmulqdq	$0x00, %xmm5, %xmm8
 2824        pxor	%xmm5, %xmm9
 2825        pxor	%xmm5, %xmm10
 2826        pclmulqdq	$0x00, %xmm10, %xmm9
 2827        pxor	%xmm8, %xmm9
 2828        pxor	%xmm11, %xmm9
 2829        movdqa	%xmm9, %xmm10
 2830        movdqa	%xmm11, %xmm0
 2831        pslldq	$8, %xmm10
 2832        psrldq	$8, %xmm9
 2833        pxor	%xmm10, %xmm8
 2834        pxor	%xmm9, %xmm0
 2835        movdqa	%xmm8, %xmm12
 2836        movdqa	%xmm8, %xmm13
 2837        movdqa	%xmm8, %xmm14
 2838        pslld	$31, %xmm12
 2839        pslld	$30, %xmm13
 2840        pslld	$25, %xmm14
 2841        pxor	%xmm13, %xmm12
 2842        pxor	%xmm14, %xmm12
 2843        movdqa	%xmm12, %xmm13
 2844        psrldq	$4, %xmm13
 2845        pslldq	$12, %xmm12
 2846        pxor	%xmm12, %xmm8
 2847        movdqa	%xmm8, %xmm14
 2848        movdqa	%xmm8, %xmm10
 2849        movdqa	%xmm8, %xmm9
 2850        psrld	$0x01, %xmm14
 2851        psrld	$2, %xmm10
 2852        psrld	$7, %xmm9
 2853        pxor	%xmm10, %xmm14
 2854        pxor	%xmm9, %xmm14
 2855        pxor	%xmm13, %xmm14
 2856        pxor	%xmm8, %xmm14
 2857        pxor	%xmm14, %xmm0
 2858        movdqu	%xmm0, 16(%rsp)
 2859        # H ^ 3
 2860        pshufd	$0x4e, %xmm5, %xmm9
 2861        pshufd	$0x4e, %xmm0, %xmm10
 2862        movdqa	%xmm0, %xmm11
 2863        movdqa	%xmm0, %xmm8
 2864        pclmulqdq	$0x11, %xmm5, %xmm11
 2865        pclmulqdq	$0x00, %xmm5, %xmm8
 2866        pxor	%xmm5, %xmm9
 2867        pxor	%xmm0, %xmm10
 2868        pclmulqdq	$0x00, %xmm10, %xmm9
 2869        pxor	%xmm8, %xmm9
 2870        pxor	%xmm11, %xmm9
 2871        movdqa	%xmm9, %xmm10
 2872        movdqa	%xmm11, %xmm1
 2873        pslldq	$8, %xmm10
 2874        psrldq	$8, %xmm9
 2875        pxor	%xmm10, %xmm8
 2876        pxor	%xmm9, %xmm1
 2877        movdqa	%xmm8, %xmm12
 2878        movdqa	%xmm8, %xmm13
 2879        movdqa	%xmm8, %xmm14
 2880        pslld	$31, %xmm12
 2881        pslld	$30, %xmm13
 2882        pslld	$25, %xmm14
 2883        pxor	%xmm13, %xmm12
 2884        pxor	%xmm14, %xmm12
 2885        movdqa	%xmm12, %xmm13
 2886        psrldq	$4, %xmm13
 2887        pslldq	$12, %xmm12
 2888        pxor	%xmm12, %xmm8
 2889        movdqa	%xmm8, %xmm14
 2890        movdqa	%xmm8, %xmm10
 2891        movdqa	%xmm8, %xmm9
 2892        psrld	$0x01, %xmm14
 2893        psrld	$2, %xmm10
 2894        psrld	$7, %xmm9
 2895        pxor	%xmm10, %xmm14
 2896        pxor	%xmm9, %xmm14
 2897        pxor	%xmm13, %xmm14
 2898        pxor	%xmm8, %xmm14
 2899        pxor	%xmm14, %xmm1
 2900        movdqu	%xmm1, 32(%rsp)
 2901        # H ^ 4
 2902        pshufd	$0x4e, %xmm0, %xmm9
 2903        pshufd	$0x4e, %xmm0, %xmm10
 2904        movdqa	%xmm0, %xmm11
 2905        movdqa	%xmm0, %xmm8
 2906        pclmulqdq	$0x11, %xmm0, %xmm11
 2907        pclmulqdq	$0x00, %xmm0, %xmm8
 2908        pxor	%xmm0, %xmm9
 2909        pxor	%xmm0, %xmm10
 2910        pclmulqdq	$0x00, %xmm10, %xmm9
 2911        pxor	%xmm8, %xmm9
 2912        pxor	%xmm11, %xmm9
 2913        movdqa	%xmm9, %xmm10
 2914        movdqa	%xmm11, %xmm3
 2915        pslldq	$8, %xmm10
 2916        psrldq	$8, %xmm9
 2917        pxor	%xmm10, %xmm8
 2918        pxor	%xmm9, %xmm3
 2919        movdqa	%xmm8, %xmm12
 2920        movdqa	%xmm8, %xmm13
 2921        movdqa	%xmm8, %xmm14
 2922        pslld	$31, %xmm12
 2923        pslld	$30, %xmm13
 2924        pslld	$25, %xmm14
 2925        pxor	%xmm13, %xmm12
 2926        pxor	%xmm14, %xmm12
 2927        movdqa	%xmm12, %xmm13
 2928        psrldq	$4, %xmm13
 2929        pslldq	$12, %xmm12
 2930        pxor	%xmm12, %xmm8
 2931        movdqa	%xmm8, %xmm14
 2932        movdqa	%xmm8, %xmm10
 2933        movdqa	%xmm8, %xmm9
 2934        psrld	$0x01, %xmm14
 2935        psrld	$2, %xmm10
 2936        psrld	$7, %xmm9
 2937        pxor	%xmm10, %xmm14
 2938        pxor	%xmm9, %xmm14
 2939        pxor	%xmm13, %xmm14
 2940        pxor	%xmm8, %xmm14
 2941        pxor	%xmm14, %xmm3
 2942        movdqu	%xmm3, 48(%rsp)
 2943        # H ^ 5
 2944        pshufd	$0x4e, %xmm0, %xmm9
 2945        pshufd	$0x4e, %xmm1, %xmm10
 2946        movdqa	%xmm1, %xmm11
 2947        movdqa	%xmm1, %xmm8
 2948        pclmulqdq	$0x11, %xmm0, %xmm11
 2949        pclmulqdq	$0x00, %xmm0, %xmm8
 2950        pxor	%xmm0, %xmm9
 2951        pxor	%xmm1, %xmm10
 2952        pclmulqdq	$0x00, %xmm10, %xmm9
 2953        pxor	%xmm8, %xmm9
 2954        pxor	%xmm11, %xmm9
 2955        movdqa	%xmm9, %xmm10
 2956        movdqa	%xmm11, %xmm7
 2957        pslldq	$8, %xmm10
 2958        psrldq	$8, %xmm9
 2959        pxor	%xmm10, %xmm8
 2960        pxor	%xmm9, %xmm7
 2961        movdqa	%xmm8, %xmm12
 2962        movdqa	%xmm8, %xmm13
 2963        movdqa	%xmm8, %xmm14
 2964        pslld	$31, %xmm12
 2965        pslld	$30, %xmm13
 2966        pslld	$25, %xmm14
 2967        pxor	%xmm13, %xmm12
 2968        pxor	%xmm14, %xmm12
 2969        movdqa	%xmm12, %xmm13
 2970        psrldq	$4, %xmm13
 2971        pslldq	$12, %xmm12
 2972        pxor	%xmm12, %xmm8
 2973        movdqa	%xmm8, %xmm14
 2974        movdqa	%xmm8, %xmm10
 2975        movdqa	%xmm8, %xmm9
 2976        psrld	$0x01, %xmm14
 2977        psrld	$2, %xmm10
 2978        psrld	$7, %xmm9
 2979        pxor	%xmm10, %xmm14
 2980        pxor	%xmm9, %xmm14
 2981        pxor	%xmm13, %xmm14
 2982        pxor	%xmm8, %xmm14
 2983        pxor	%xmm14, %xmm7
 2984        movdqu	%xmm7, 64(%rsp)
 2985        # H ^ 6
 2986        pshufd	$0x4e, %xmm1, %xmm9
 2987        pshufd	$0x4e, %xmm1, %xmm10
 2988        movdqa	%xmm1, %xmm11
 2989        movdqa	%xmm1, %xmm8
 2990        pclmulqdq	$0x11, %xmm1, %xmm11
 2991        pclmulqdq	$0x00, %xmm1, %xmm8
 2992        pxor	%xmm1, %xmm9
 2993        pxor	%xmm1, %xmm10
 2994        pclmulqdq	$0x00, %xmm10, %xmm9
 2995        pxor	%xmm8, %xmm9
 2996        pxor	%xmm11, %xmm9
 2997        movdqa	%xmm9, %xmm10
 2998        movdqa	%xmm11, %xmm7
 2999        pslldq	$8, %xmm10
 3000        psrldq	$8, %xmm9
 3001        pxor	%xmm10, %xmm8
 3002        pxor	%xmm9, %xmm7
 3003        movdqa	%xmm8, %xmm12
 3004        movdqa	%xmm8, %xmm13
 3005        movdqa	%xmm8, %xmm14
 3006        pslld	$31, %xmm12
 3007        pslld	$30, %xmm13
 3008        pslld	$25, %xmm14
 3009        pxor	%xmm13, %xmm12
 3010        pxor	%xmm14, %xmm12
 3011        movdqa	%xmm12, %xmm13
 3012        psrldq	$4, %xmm13
 3013        pslldq	$12, %xmm12
 3014        pxor	%xmm12, %xmm8
 3015        movdqa	%xmm8, %xmm14
 3016        movdqa	%xmm8, %xmm10
 3017        movdqa	%xmm8, %xmm9
 3018        psrld	$0x01, %xmm14
 3019        psrld	$2, %xmm10
 3020        psrld	$7, %xmm9
 3021        pxor	%xmm10, %xmm14
 3022        pxor	%xmm9, %xmm14
 3023        pxor	%xmm13, %xmm14
 3024        pxor	%xmm8, %xmm14
 3025        pxor	%xmm14, %xmm7
 3026        movdqu	%xmm7, 80(%rsp)
 3027        # H ^ 7
 3028        pshufd	$0x4e, %xmm1, %xmm9
 3029        pshufd	$0x4e, %xmm3, %xmm10
 3030        movdqa	%xmm3, %xmm11
 3031        movdqa	%xmm3, %xmm8
 3032        pclmulqdq	$0x11, %xmm1, %xmm11
 3033        pclmulqdq	$0x00, %xmm1, %xmm8
 3034        pxor	%xmm1, %xmm9
 3035        pxor	%xmm3, %xmm10
 3036        pclmulqdq	$0x00, %xmm10, %xmm9
 3037        pxor	%xmm8, %xmm9
 3038        pxor	%xmm11, %xmm9
 3039        movdqa	%xmm9, %xmm10
 3040        movdqa	%xmm11, %xmm7
 3041        pslldq	$8, %xmm10
 3042        psrldq	$8, %xmm9
 3043        pxor	%xmm10, %xmm8
 3044        pxor	%xmm9, %xmm7
 3045        movdqa	%xmm8, %xmm12
 3046        movdqa	%xmm8, %xmm13
 3047        movdqa	%xmm8, %xmm14
 3048        pslld	$31, %xmm12
 3049        pslld	$30, %xmm13
 3050        pslld	$25, %xmm14
 3051        pxor	%xmm13, %xmm12
 3052        pxor	%xmm14, %xmm12
 3053        movdqa	%xmm12, %xmm13
 3054        psrldq	$4, %xmm13
 3055        pslldq	$12, %xmm12
 3056        pxor	%xmm12, %xmm8
 3057        movdqa	%xmm8, %xmm14
 3058        movdqa	%xmm8, %xmm10
 3059        movdqa	%xmm8, %xmm9
 3060        psrld	$0x01, %xmm14
 3061        psrld	$2, %xmm10
 3062        psrld	$7, %xmm9
 3063        pxor	%xmm10, %xmm14
 3064        pxor	%xmm9, %xmm14
 3065        pxor	%xmm13, %xmm14
 3066        pxor	%xmm8, %xmm14
 3067        pxor	%xmm14, %xmm7
 3068        movdqu	%xmm7, 96(%rsp)
 3069        # H ^ 8
 3070        pshufd	$0x4e, %xmm3, %xmm9
 3071        pshufd	$0x4e, %xmm3, %xmm10
 3072        movdqa	%xmm3, %xmm11
 3073        movdqa	%xmm3, %xmm8
 3074        pclmulqdq	$0x11, %xmm3, %xmm11
 3075        pclmulqdq	$0x00, %xmm3, %xmm8
 3076        pxor	%xmm3, %xmm9
 3077        pxor	%xmm3, %xmm10
 3078        pclmulqdq	$0x00, %xmm10, %xmm9
 3079        pxor	%xmm8, %xmm9
 3080        pxor	%xmm11, %xmm9
 3081        movdqa	%xmm9, %xmm10
 3082        movdqa	%xmm11, %xmm7
 3083        pslldq	$8, %xmm10
 3084        psrldq	$8, %xmm9
 3085        pxor	%xmm10, %xmm8
 3086        pxor	%xmm9, %xmm7
 3087        movdqa	%xmm8, %xmm12
 3088        movdqa	%xmm8, %xmm13
 3089        movdqa	%xmm8, %xmm14
 3090        pslld	$31, %xmm12
 3091        pslld	$30, %xmm13
 3092        pslld	$25, %xmm14
 3093        pxor	%xmm13, %xmm12
 3094        pxor	%xmm14, %xmm12
 3095        movdqa	%xmm12, %xmm13
 3096        psrldq	$4, %xmm13
 3097        pslldq	$12, %xmm12
 3098        pxor	%xmm12, %xmm8
 3099        movdqa	%xmm8, %xmm14
 3100        movdqa	%xmm8, %xmm10
 3101        movdqa	%xmm8, %xmm9
 3102        psrld	$0x01, %xmm14
 3103        psrld	$2, %xmm10
 3104        psrld	$7, %xmm9
 3105        pxor	%xmm10, %xmm14
 3106        pxor	%xmm9, %xmm14
 3107        pxor	%xmm13, %xmm14
 3108        pxor	%xmm8, %xmm14
 3109        pxor	%xmm14, %xmm7
 3110        movdqu	%xmm7, 112(%rsp)
 3111L_AES_GCM_decrypt_aesni_ghash_128:
 3112        leaq	(%rdi,%rbx,1), %rcx
 3113        leaq	(%rsi,%rbx,1), %rdx
 3114        movdqu	128(%rsp), %xmm8
 3115        movdqa	L_aes_gcm_bswap_epi64(%rip), %xmm1
 3116        movdqa	%xmm8, %xmm0
 3117        pshufb	%xmm1, %xmm8
 3118        movdqa	%xmm0, %xmm9
 3119        paddd	L_aes_gcm_one(%rip), %xmm9
 3120        pshufb	%xmm1, %xmm9
 3121        movdqa	%xmm0, %xmm10
 3122        paddd	L_aes_gcm_two(%rip), %xmm10
 3123        pshufb	%xmm1, %xmm10
 3124        movdqa	%xmm0, %xmm11
 3125        paddd	L_aes_gcm_three(%rip), %xmm11
 3126        pshufb	%xmm1, %xmm11
 3127        movdqa	%xmm0, %xmm12
 3128        paddd	L_aes_gcm_four(%rip), %xmm12
 3129        pshufb	%xmm1, %xmm12
 3130        movdqa	%xmm0, %xmm13
 3131        paddd	L_aes_gcm_five(%rip), %xmm13
 3132        pshufb	%xmm1, %xmm13
 3133        movdqa	%xmm0, %xmm14
 3134        paddd	L_aes_gcm_six(%rip), %xmm14
 3135        pshufb	%xmm1, %xmm14
 3136        movdqa	%xmm0, %xmm15
 3137        paddd	L_aes_gcm_seven(%rip), %xmm15
 3138        pshufb	%xmm1, %xmm15
 3139        paddd	L_aes_gcm_eight(%rip), %xmm0
 3140        movdqa	(%r15), %xmm7
 3141        movdqu	%xmm0, 128(%rsp)
 3142        pxor	%xmm7, %xmm8
 3143        pxor	%xmm7, %xmm9
 3144        pxor	%xmm7, %xmm10
 3145        pxor	%xmm7, %xmm11
 3146        pxor	%xmm7, %xmm12
 3147        pxor	%xmm7, %xmm13
 3148        pxor	%xmm7, %xmm14
 3149        pxor	%xmm7, %xmm15
 3150        movdqu	112(%rsp), %xmm7
 3151        movdqu	(%rcx), %xmm0
 3152        aesenc	16(%r15), %xmm8
 3153        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm0
 3154        pxor	%xmm2, %xmm0
 3155        pshufd	$0x4e, %xmm7, %xmm1
 3156        pshufd	$0x4e, %xmm0, %xmm5
 3157        pxor	%xmm7, %xmm1
 3158        pxor	%xmm0, %xmm5
 3159        movdqa	%xmm0, %xmm3
 3160        pclmulqdq	$0x11, %xmm7, %xmm3
 3161        aesenc	16(%r15), %xmm9
 3162        aesenc	16(%r15), %xmm10
 3163        movdqa	%xmm0, %xmm2
 3164        pclmulqdq	$0x00, %xmm7, %xmm2
 3165        aesenc	16(%r15), %xmm11
 3166        aesenc	16(%r15), %xmm12
 3167        pclmulqdq	$0x00, %xmm5, %xmm1
 3168        aesenc	16(%r15), %xmm13
 3169        aesenc	16(%r15), %xmm14
 3170        aesenc	16(%r15), %xmm15
 3171        pxor	%xmm2, %xmm1
 3172        pxor	%xmm3, %xmm1
 3173        movdqu	96(%rsp), %xmm7
 3174        movdqu	16(%rcx), %xmm0
 3175        pshufd	$0x4e, %xmm7, %xmm4
 3176        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm0
 3177        aesenc	32(%r15), %xmm8
 3178        pxor	%xmm7, %xmm4
 3179        pshufd	$0x4e, %xmm0, %xmm5
 3180        pxor	%xmm0, %xmm5
 3181        movdqa	%xmm0, %xmm6
 3182        pclmulqdq	$0x11, %xmm7, %xmm6
 3183        aesenc	32(%r15), %xmm9
 3184        aesenc	32(%r15), %xmm10
 3185        pclmulqdq	$0x00, %xmm0, %xmm7
 3186        aesenc	32(%r15), %xmm11
 3187        aesenc	32(%r15), %xmm12
 3188        pclmulqdq	$0x00, %xmm5, %xmm4
 3189        aesenc	32(%r15), %xmm13
 3190        aesenc	32(%r15), %xmm14
 3191        aesenc	32(%r15), %xmm15
 3192        pxor	%xmm7, %xmm1
 3193        pxor	%xmm7, %xmm2
 3194        pxor	%xmm6, %xmm1
 3195        pxor	%xmm6, %xmm3
 3196        pxor	%xmm4, %xmm1
 3197        movdqu	80(%rsp), %xmm7
 3198        movdqu	32(%rcx), %xmm0
 3199        pshufd	$0x4e, %xmm7, %xmm4
 3200        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm0
 3201        aesenc	48(%r15), %xmm8
 3202        pxor	%xmm7, %xmm4
 3203        pshufd	$0x4e, %xmm0, %xmm5
 3204        pxor	%xmm0, %xmm5
 3205        movdqa	%xmm0, %xmm6
 3206        pclmulqdq	$0x11, %xmm7, %xmm6
 3207        aesenc	48(%r15), %xmm9
 3208        aesenc	48(%r15), %xmm10
 3209        pclmulqdq	$0x00, %xmm0, %xmm7
 3210        aesenc	48(%r15), %xmm11
 3211        aesenc	48(%r15), %xmm12
 3212        pclmulqdq	$0x00, %xmm5, %xmm4
 3213        aesenc	48(%r15), %xmm13
 3214        aesenc	48(%r15), %xmm14
 3215        aesenc	48(%r15), %xmm15
 3216        pxor	%xmm7, %xmm1
 3217        pxor	%xmm7, %xmm2
 3218        pxor	%xmm6, %xmm1
 3219        pxor	%xmm6, %xmm3
 3220        pxor	%xmm4, %xmm1
 3221        movdqu	64(%rsp), %xmm7
 3222        movdqu	48(%rcx), %xmm0
 3223        pshufd	$0x4e, %xmm7, %xmm4
 3224        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm0
 3225        aesenc	64(%r15), %xmm8
 3226        pxor	%xmm7, %xmm4
 3227        pshufd	$0x4e, %xmm0, %xmm5
 3228        pxor	%xmm0, %xmm5
 3229        movdqa	%xmm0, %xmm6
 3230        pclmulqdq	$0x11, %xmm7, %xmm6
 3231        aesenc	64(%r15), %xmm9
 3232        aesenc	64(%r15), %xmm10
 3233        pclmulqdq	$0x00, %xmm0, %xmm7
 3234        aesenc	64(%r15), %xmm11
 3235        aesenc	64(%r15), %xmm12
 3236        pclmulqdq	$0x00, %xmm5, %xmm4
 3237        aesenc	64(%r15), %xmm13
 3238        aesenc	64(%r15), %xmm14
 3239        aesenc	64(%r15), %xmm15
 3240        pxor	%xmm7, %xmm1
 3241        pxor	%xmm7, %xmm2
 3242        pxor	%xmm6, %xmm1
 3243        pxor	%xmm6, %xmm3
 3244        pxor	%xmm4, %xmm1
 3245        movdqu	48(%rsp), %xmm7
 3246        movdqu	64(%rcx), %xmm0
 3247        pshufd	$0x4e, %xmm7, %xmm4
 3248        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm0
 3249        aesenc	80(%r15), %xmm8
 3250        pxor	%xmm7, %xmm4
 3251        pshufd	$0x4e, %xmm0, %xmm5
 3252        pxor	%xmm0, %xmm5
 3253        movdqa	%xmm0, %xmm6
 3254        pclmulqdq	$0x11, %xmm7, %xmm6
 3255        aesenc	80(%r15), %xmm9
 3256        aesenc	80(%r15), %xmm10
 3257        pclmulqdq	$0x00, %xmm0, %xmm7
 3258        aesenc	80(%r15), %xmm11
 3259        aesenc	80(%r15), %xmm12
 3260        pclmulqdq	$0x00, %xmm5, %xmm4
 3261        aesenc	80(%r15), %xmm13
 3262        aesenc	80(%r15), %xmm14
 3263        aesenc	80(%r15), %xmm15
 3264        pxor	%xmm7, %xmm1
 3265        pxor	%xmm7, %xmm2
 3266        pxor	%xmm6, %xmm1
 3267        pxor	%xmm6, %xmm3
 3268        pxor	%xmm4, %xmm1
 3269        movdqu	32(%rsp), %xmm7
 3270        movdqu	80(%rcx), %xmm0
 3271        pshufd	$0x4e, %xmm7, %xmm4
 3272        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm0
 3273        aesenc	96(%r15), %xmm8
 3274        pxor	%xmm7, %xmm4
 3275        pshufd	$0x4e, %xmm0, %xmm5
 3276        pxor	%xmm0, %xmm5
 3277        movdqa	%xmm0, %xmm6
 3278        pclmulqdq	$0x11, %xmm7, %xmm6
 3279        aesenc	96(%r15), %xmm9
 3280        aesenc	96(%r15), %xmm10
 3281        pclmulqdq	$0x00, %xmm0, %xmm7
 3282        aesenc	96(%r15), %xmm11
 3283        aesenc	96(%r15), %xmm12
 3284        pclmulqdq	$0x00, %xmm5, %xmm4
 3285        aesenc	96(%r15), %xmm13
 3286        aesenc	96(%r15), %xmm14
 3287        aesenc	96(%r15), %xmm15
 3288        pxor	%xmm7, %xmm1
 3289        pxor	%xmm7, %xmm2
 3290        pxor	%xmm6, %xmm1
 3291        pxor	%xmm6, %xmm3
 3292        pxor	%xmm4, %xmm1
 3293        movdqu	16(%rsp), %xmm7
 3294        movdqu	96(%rcx), %xmm0
 3295        pshufd	$0x4e, %xmm7, %xmm4
 3296        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm0
 3297        aesenc	112(%r15), %xmm8
 3298        pxor	%xmm7, %xmm4
 3299        pshufd	$0x4e, %xmm0, %xmm5
 3300        pxor	%xmm0, %xmm5
 3301        movdqa	%xmm0, %xmm6
 3302        pclmulqdq	$0x11, %xmm7, %xmm6
 3303        aesenc	112(%r15), %xmm9
 3304        aesenc	112(%r15), %xmm10
 3305        pclmulqdq	$0x00, %xmm0, %xmm7
 3306        aesenc	112(%r15), %xmm11
 3307        aesenc	112(%r15), %xmm12
 3308        pclmulqdq	$0x00, %xmm5, %xmm4
 3309        aesenc	112(%r15), %xmm13
 3310        aesenc	112(%r15), %xmm14
 3311        aesenc	112(%r15), %xmm15
 3312        pxor	%xmm7, %xmm1
 3313        pxor	%xmm7, %xmm2
 3314        pxor	%xmm6, %xmm1
 3315        pxor	%xmm6, %xmm3
 3316        pxor	%xmm4, %xmm1
 3317        movdqu	(%rsp), %xmm7
 3318        movdqu	112(%rcx), %xmm0
 3319        pshufd	$0x4e, %xmm7, %xmm4
 3320        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm0
 3321        aesenc	128(%r15), %xmm8
 3322        pxor	%xmm7, %xmm4
 3323        pshufd	$0x4e, %xmm0, %xmm5
 3324        pxor	%xmm0, %xmm5
 3325        movdqa	%xmm0, %xmm6
 3326        pclmulqdq	$0x11, %xmm7, %xmm6
 3327        aesenc	128(%r15), %xmm9
 3328        aesenc	128(%r15), %xmm10
 3329        pclmulqdq	$0x00, %xmm0, %xmm7
 3330        aesenc	128(%r15), %xmm11
 3331        aesenc	128(%r15), %xmm12
 3332        pclmulqdq	$0x00, %xmm5, %xmm4
 3333        aesenc	128(%r15), %xmm13
 3334        aesenc	128(%r15), %xmm14
 3335        aesenc	128(%r15), %xmm15
 3336        pxor	%xmm7, %xmm1
 3337        pxor	%xmm7, %xmm2
 3338        pxor	%xmm6, %xmm1
 3339        pxor	%xmm6, %xmm3
 3340        pxor	%xmm4, %xmm1
 3341        movdqa	%xmm1, %xmm5
 3342        psrldq	$8, %xmm1
 3343        pslldq	$8, %xmm5
 3344        aesenc	144(%r15), %xmm8
 3345        pxor	%xmm5, %xmm2
 3346        pxor	%xmm1, %xmm3
 3347        movdqa	%xmm2, %xmm7
 3348        movdqa	%xmm2, %xmm4
 3349        movdqa	%xmm2, %xmm5
 3350        aesenc	144(%r15), %xmm9
 3351        pslld	$31, %xmm7
 3352        pslld	$30, %xmm4
 3353        pslld	$25, %xmm5
 3354        aesenc	144(%r15), %xmm10
 3355        pxor	%xmm4, %xmm7
 3356        pxor	%xmm5, %xmm7
 3357        aesenc	144(%r15), %xmm11
 3358        movdqa	%xmm7, %xmm4
 3359        pslldq	$12, %xmm7
 3360        psrldq	$4, %xmm4
 3361        aesenc	144(%r15), %xmm12
 3362        pxor	%xmm7, %xmm2
 3363        movdqa	%xmm2, %xmm5
 3364        movdqa	%xmm2, %xmm1
 3365        movdqa	%xmm2, %xmm0
 3366        aesenc	144(%r15), %xmm13
 3367        psrld	$0x01, %xmm5
 3368        psrld	$2, %xmm1
 3369        psrld	$7, %xmm0
 3370        aesenc	144(%r15), %xmm14
 3371        pxor	%xmm1, %xmm5
 3372        pxor	%xmm0, %xmm5
 3373        aesenc	144(%r15), %xmm15
 3374        pxor	%xmm4, %xmm5
 3375        pxor	%xmm5, %xmm2
 3376        pxor	%xmm3, %xmm2
 3377        cmpl	$11, %r10d
 3378        movdqa	160(%r15), %xmm7
 3379        jl	L_AES_GCM_decrypt_aesni_aesenc_128_ghash_avx_done
 3380        aesenc	%xmm7, %xmm8
 3381        aesenc	%xmm7, %xmm9
 3382        aesenc	%xmm7, %xmm10
 3383        aesenc	%xmm7, %xmm11
 3384        aesenc	%xmm7, %xmm12
 3385        aesenc	%xmm7, %xmm13
 3386        aesenc	%xmm7, %xmm14
 3387        aesenc	%xmm7, %xmm15
 3388        movdqa	176(%r15), %xmm7
 3389        aesenc	%xmm7, %xmm8
 3390        aesenc	%xmm7, %xmm9
 3391        aesenc	%xmm7, %xmm10
 3392        aesenc	%xmm7, %xmm11
 3393        aesenc	%xmm7, %xmm12
 3394        aesenc	%xmm7, %xmm13
 3395        aesenc	%xmm7, %xmm14
 3396        aesenc	%xmm7, %xmm15
 3397        cmpl	$13, %r10d
 3398        movdqa	192(%r15), %xmm7
 3399        jl	L_AES_GCM_decrypt_aesni_aesenc_128_ghash_avx_done
 3400        aesenc	%xmm7, %xmm8
 3401        aesenc	%xmm7, %xmm9
 3402        aesenc	%xmm7, %xmm10
 3403        aesenc	%xmm7, %xmm11
 3404        aesenc	%xmm7, %xmm12
 3405        aesenc	%xmm7, %xmm13
 3406        aesenc	%xmm7, %xmm14
 3407        aesenc	%xmm7, %xmm15
 3408        movdqa	208(%r15), %xmm7
 3409        aesenc	%xmm7, %xmm8
 3410        aesenc	%xmm7, %xmm9
 3411        aesenc	%xmm7, %xmm10
 3412        aesenc	%xmm7, %xmm11
 3413        aesenc	%xmm7, %xmm12
 3414        aesenc	%xmm7, %xmm13
 3415        aesenc	%xmm7, %xmm14
 3416        aesenc	%xmm7, %xmm15
 3417        movdqa	224(%r15), %xmm7
 3418L_AES_GCM_decrypt_aesni_aesenc_128_ghash_avx_done:
 3419        aesenclast	%xmm7, %xmm8
 3420        aesenclast	%xmm7, %xmm9
 3421        movdqu	(%rcx), %xmm0
 3422        movdqu	16(%rcx), %xmm1
 3423        pxor	%xmm0, %xmm8
 3424        pxor	%xmm1, %xmm9
 3425        movdqu	%xmm8, (%rdx)
 3426        movdqu	%xmm9, 16(%rdx)
 3427        aesenclast	%xmm7, %xmm10
 3428        aesenclast	%xmm7, %xmm11
 3429        movdqu	32(%rcx), %xmm0
 3430        movdqu	48(%rcx), %xmm1
 3431        pxor	%xmm0, %xmm10
 3432        pxor	%xmm1, %xmm11
 3433        movdqu	%xmm10, 32(%rdx)
 3434        movdqu	%xmm11, 48(%rdx)
 3435        aesenclast	%xmm7, %xmm12
 3436        aesenclast	%xmm7, %xmm13
 3437        movdqu	64(%rcx), %xmm0
 3438        movdqu	80(%rcx), %xmm1
 3439        pxor	%xmm0, %xmm12
 3440        pxor	%xmm1, %xmm13
 3441        movdqu	%xmm12, 64(%rdx)
 3442        movdqu	%xmm13, 80(%rdx)
 3443        aesenclast	%xmm7, %xmm14
 3444        aesenclast	%xmm7, %xmm15
 3445        movdqu	96(%rcx), %xmm0
 3446        movdqu	112(%rcx), %xmm1
 3447        pxor	%xmm0, %xmm14
 3448        pxor	%xmm1, %xmm15
 3449        movdqu	%xmm14, 96(%rdx)
 3450        movdqu	%xmm15, 112(%rdx)
 3451        addl	$0x80, %ebx
 3452        cmpl	%r13d, %ebx
 3453        jl	L_AES_GCM_decrypt_aesni_ghash_128
 3454        movdqa	%xmm2, %xmm6
 3455        movdqu	(%rsp), %xmm5
 3456L_AES_GCM_decrypt_aesni_done_128:
 3457        movl	%r9d, %edx
 3458        cmpl	%edx, %ebx
 3459        jge	L_AES_GCM_decrypt_aesni_done_dec
 3460        movl	%r9d, %r13d
 3461        andl	$0xfffffff0, %r13d
 3462        cmpl	%r13d, %ebx
 3463        jge	L_AES_GCM_decrypt_aesni_last_block_done
 3464L_AES_GCM_decrypt_aesni_last_block_start:
 3465        leaq	(%rdi,%rbx,1), %rcx
 3466        leaq	(%rsi,%rbx,1), %rdx
 3467        movdqu	(%rcx), %xmm1
 3468        movdqa	%xmm5, %xmm0
 3469        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm1
 3470        pxor	%xmm6, %xmm1
 3471        movdqu	128(%rsp), %xmm8
 3472        movdqa	%xmm8, %xmm9
 3473        pshufb	L_aes_gcm_bswap_epi64(%rip), %xmm8
 3474        paddd	L_aes_gcm_one(%rip), %xmm9
 3475        pxor	(%r15), %xmm8
 3476        movdqu	%xmm9, 128(%rsp)
 3477        movdqa	%xmm1, %xmm10
 3478        pclmulqdq	$16, %xmm0, %xmm10
 3479        aesenc	16(%r15), %xmm8
 3480        aesenc	32(%r15), %xmm8
 3481        movdqa	%xmm1, %xmm11
 3482        pclmulqdq	$0x01, %xmm0, %xmm11
 3483        aesenc	48(%r15), %xmm8
 3484        aesenc	64(%r15), %xmm8
 3485        movdqa	%xmm1, %xmm12
 3486        pclmulqdq	$0x00, %xmm0, %xmm12
 3487        aesenc	80(%r15), %xmm8
 3488        movdqa	%xmm1, %xmm1
 3489        pclmulqdq	$0x11, %xmm0, %xmm1
 3490        aesenc	96(%r15), %xmm8
 3491        pxor	%xmm11, %xmm10
 3492        movdqa	%xmm10, %xmm2
 3493        psrldq	$8, %xmm10
 3494        pslldq	$8, %xmm2
 3495        aesenc	112(%r15), %xmm8
 3496        movdqa	%xmm1, %xmm3
 3497        pxor	%xmm12, %xmm2
 3498        pxor	%xmm10, %xmm3
 3499        movdqa	L_aes_gcm_mod2_128(%rip), %xmm0
 3500        movdqa	%xmm2, %xmm11
 3501        pclmulqdq	$16, %xmm0, %xmm11
 3502        aesenc	128(%r15), %xmm8
 3503        pshufd	$0x4e, %xmm2, %xmm10
 3504        pxor	%xmm11, %xmm10
 3505        movdqa	%xmm10, %xmm11
 3506        pclmulqdq	$16, %xmm0, %xmm11
 3507        aesenc	144(%r15), %xmm8
 3508        pshufd	$0x4e, %xmm10, %xmm6
 3509        pxor	%xmm11, %xmm6
 3510        pxor	%xmm3, %xmm6
 3511        cmpl	$11, %r10d
 3512        movdqa	160(%r15), %xmm9
 3513        jl	L_AES_GCM_decrypt_aesni_aesenc_gfmul_last
 3514        aesenc	%xmm9, %xmm8
 3515        aesenc	176(%r15), %xmm8
 3516        cmpl	$13, %r10d
 3517        movdqa	192(%r15), %xmm9
 3518        jl	L_AES_GCM_decrypt_aesni_aesenc_gfmul_last
 3519        aesenc	%xmm9, %xmm8
 3520        aesenc	208(%r15), %xmm8
 3521        movdqa	224(%r15), %xmm9
 3522L_AES_GCM_decrypt_aesni_aesenc_gfmul_last:
 3523        aesenclast	%xmm9, %xmm8
 3524        movdqu	(%rcx), %xmm9
 3525        pxor	%xmm9, %xmm8
 3526        movdqu	%xmm8, (%rdx)
 3527        addl	$16, %ebx
 3528        cmpl	%r13d, %ebx
 3529        jl	L_AES_GCM_decrypt_aesni_last_block_start
 3530L_AES_GCM_decrypt_aesni_last_block_done:
 3531        movl	%r9d, %ecx
 3532        movl	%ecx, %edx
 3533        andl	$15, %ecx
 3534        jz	L_AES_GCM_decrypt_aesni_aesenc_last15_dec_avx_done
 3535        movdqu	128(%rsp), %xmm4
 3536        pshufb	L_aes_gcm_bswap_epi64(%rip), %xmm4
 3537        pxor	(%r15), %xmm4
 3538        aesenc	16(%r15), %xmm4
 3539        aesenc	32(%r15), %xmm4
 3540        aesenc	48(%r15), %xmm4
 3541        aesenc	64(%r15), %xmm4
 3542        aesenc	80(%r15), %xmm4
 3543        aesenc	96(%r15), %xmm4
 3544        aesenc	112(%r15), %xmm4
 3545        aesenc	128(%r15), %xmm4
 3546        aesenc	144(%r15), %xmm4
 3547        cmpl	$11, %r10d
 3548        movdqa	160(%r15), %xmm9
 3549        jl	L_AES_GCM_decrypt_aesni_aesenc_last15_dec_avx_aesenc_avx_last
 3550        aesenc	%xmm9, %xmm4
 3551        aesenc	176(%r15), %xmm4
 3552        cmpl	$13, %r10d
 3553        movdqa	192(%r15), %xmm9
 3554        jl	L_AES_GCM_decrypt_aesni_aesenc_last15_dec_avx_aesenc_avx_last
 3555        aesenc	%xmm9, %xmm4
 3556        aesenc	208(%r15), %xmm4
 3557        movdqa	224(%r15), %xmm9
 3558L_AES_GCM_decrypt_aesni_aesenc_last15_dec_avx_aesenc_avx_last:
 3559        aesenclast	%xmm9, %xmm4
 3560        subq	$32, %rsp
 3561        xorl	%ecx, %ecx
 3562        movdqu	%xmm4, (%rsp)
 3563        pxor	%xmm0, %xmm0
 3564        movdqu	%xmm0, 16(%rsp)
 3565L_AES_GCM_decrypt_aesni_aesenc_last15_dec_avx_loop:
 3566        movzbl	(%rdi,%rbx,1), %r13d
 3567        movb	%r13b, 16(%rsp,%rcx,1)
 3568        xorb	(%rsp,%rcx,1), %r13b
 3569        movb	%r13b, (%rsi,%rbx,1)
 3570        incl	%ebx
 3571        incl	%ecx
 3572        cmpl	%edx, %ebx
 3573        jl	L_AES_GCM_decrypt_aesni_aesenc_last15_dec_avx_loop
 3574        movdqu	16(%rsp), %xmm4
 3575        addq	$32, %rsp
 3576        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm4
 3577        pxor	%xmm4, %xmm6
 3578        pshufd	$0x4e, %xmm5, %xmm9
 3579        pshufd	$0x4e, %xmm6, %xmm10
 3580        movdqa	%xmm6, %xmm11
 3581        movdqa	%xmm6, %xmm8
 3582        pclmulqdq	$0x11, %xmm5, %xmm11
 3583        pclmulqdq	$0x00, %xmm5, %xmm8
 3584        pxor	%xmm5, %xmm9
 3585        pxor	%xmm6, %xmm10
 3586        pclmulqdq	$0x00, %xmm10, %xmm9
 3587        pxor	%xmm8, %xmm9
 3588        pxor	%xmm11, %xmm9
 3589        movdqa	%xmm9, %xmm10
 3590        movdqa	%xmm11, %xmm6
 3591        pslldq	$8, %xmm10
 3592        psrldq	$8, %xmm9
 3593        pxor	%xmm10, %xmm8
 3594        pxor	%xmm9, %xmm6
 3595        movdqa	%xmm8, %xmm12
 3596        movdqa	%xmm8, %xmm13
 3597        movdqa	%xmm8, %xmm14
 3598        pslld	$31, %xmm12
 3599        pslld	$30, %xmm13
 3600        pslld	$25, %xmm14
 3601        pxor	%xmm13, %xmm12
 3602        pxor	%xmm14, %xmm12
 3603        movdqa	%xmm12, %xmm13
 3604        psrldq	$4, %xmm13
 3605        pslldq	$12, %xmm12
 3606        pxor	%xmm12, %xmm8
 3607        movdqa	%xmm8, %xmm14
 3608        movdqa	%xmm8, %xmm10
 3609        movdqa	%xmm8, %xmm9
 3610        psrld	$0x01, %xmm14
 3611        psrld	$2, %xmm10
 3612        psrld	$7, %xmm9
 3613        pxor	%xmm10, %xmm14
 3614        pxor	%xmm9, %xmm14
 3615        pxor	%xmm13, %xmm14
 3616        pxor	%xmm8, %xmm14
 3617        pxor	%xmm14, %xmm6
 3618L_AES_GCM_decrypt_aesni_aesenc_last15_dec_avx_done:
 3619L_AES_GCM_decrypt_aesni_done_dec:
 3620        movl	%r9d, %edx
 3621        movl	%r11d, %ecx
 3622        shlq	$3, %rdx
 3623        shlq	$3, %rcx
 3624        pinsrq	$0x00, %rdx, %xmm0
 3625        pinsrq	$0x01, %rcx, %xmm0
 3626        pxor	%xmm0, %xmm6
 3627        pshufd	$0x4e, %xmm5, %xmm9
 3628        pshufd	$0x4e, %xmm6, %xmm10
 3629        movdqa	%xmm6, %xmm11
 3630        movdqa	%xmm6, %xmm8
 3631        pclmulqdq	$0x11, %xmm5, %xmm11
 3632        pclmulqdq	$0x00, %xmm5, %xmm8
 3633        pxor	%xmm5, %xmm9
 3634        pxor	%xmm6, %xmm10
 3635        pclmulqdq	$0x00, %xmm10, %xmm9
 3636        pxor	%xmm8, %xmm9
 3637        pxor	%xmm11, %xmm9
 3638        movdqa	%xmm9, %xmm10
 3639        movdqa	%xmm11, %xmm6
 3640        pslldq	$8, %xmm10
 3641        psrldq	$8, %xmm9
 3642        pxor	%xmm10, %xmm8
 3643        pxor	%xmm9, %xmm6
 3644        movdqa	%xmm8, %xmm12
 3645        movdqa	%xmm8, %xmm13
 3646        movdqa	%xmm8, %xmm14
 3647        pslld	$31, %xmm12
 3648        pslld	$30, %xmm13
 3649        pslld	$25, %xmm14
 3650        pxor	%xmm13, %xmm12
 3651        pxor	%xmm14, %xmm12
 3652        movdqa	%xmm12, %xmm13
 3653        psrldq	$4, %xmm13
 3654        pslldq	$12, %xmm12
 3655        pxor	%xmm12, %xmm8
 3656        movdqa	%xmm8, %xmm14
 3657        movdqa	%xmm8, %xmm10
 3658        movdqa	%xmm8, %xmm9
 3659        psrld	$0x01, %xmm14
 3660        psrld	$2, %xmm10
 3661        psrld	$7, %xmm9
 3662        pxor	%xmm10, %xmm14
 3663        pxor	%xmm9, %xmm14
 3664        pxor	%xmm13, %xmm14
 3665        pxor	%xmm8, %xmm14
 3666        pxor	%xmm14, %xmm6
 3667        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm6
 3668        movdqu	144(%rsp), %xmm0
 3669        pxor	%xmm6, %xmm0
 3670        cmpl	$16, %r14d
 3671        je	L_AES_GCM_decrypt_aesni_cmp_tag_16
 3672        subq	$16, %rsp
 3673        xorq	%rcx, %rcx
 3674        xorq	%rbx, %rbx
 3675        movdqu	%xmm0, (%rsp)
 3676L_AES_GCM_decrypt_aesni_cmp_tag_loop:
 3677        movzbl	(%rsp,%rcx,1), %r13d
 3678        xorb	(%r8,%rcx,1), %r13b
 3679        orb	%r13b, %bl
 3680        incl	%ecx
 3681        cmpl	%r14d, %ecx
 3682        jne	L_AES_GCM_decrypt_aesni_cmp_tag_loop
 3683        cmpb	$0x00, %bl
 3684        sete	%bl
 3685        addq	$16, %rsp
 3686        xorq	%rcx, %rcx
 3687        jmp	L_AES_GCM_decrypt_aesni_cmp_tag_done
 3688L_AES_GCM_decrypt_aesni_cmp_tag_16:
 3689        movdqu	(%r8), %xmm1
 3690        pcmpeqb	%xmm1, %xmm0
 3691        pmovmskb	%xmm0, %rdx
 3692        # %%edx == 0xFFFF then return 1 else => return 0
 3693        xorl	%ebx, %ebx
 3694        cmpl	$0xffff, %edx
 3695        sete	%bl
 3696L_AES_GCM_decrypt_aesni_cmp_tag_done:
 3697        movl	%ebx, (%rbp)
 3698        addq	$0xa8, %rsp
 3699        popq	%rbp
 3700        popq	%r15
 3701        popq	%r14
 3702        popq	%rbx
 3703        popq	%r12
 3704        popq	%r13
 3705        repz retq
 3706#ifndef __APPLE__
 3707.size	AES_GCM_decrypt_aesni,.-AES_GCM_decrypt_aesni
 3708#endif /* __APPLE__ */
 3709#ifdef WOLFSSL_AESGCM_STREAM
 3710#ifndef __APPLE__
 3711.text
 3712.globl	AES_GCM_init_aesni
 3713.type	AES_GCM_init_aesni,@function
 3714.align	16
 3715AES_GCM_init_aesni:
 3716#else
 3717.section	__TEXT,__text
 3718.globl	_AES_GCM_init_aesni
 3719.p2align	4
 3720_AES_GCM_init_aesni:
 3721#endif /* __APPLE__ */
 3722        pushq	%r12
 3723        pushq	%r13
 3724        pushq	%r14
 3725        movq	%rdx, %r10
 3726        movl	%ecx, %r11d
 3727        movq	32(%rsp), %rax
 3728        subq	$16, %rsp
 3729        pxor	%xmm4, %xmm4
 3730        movl	%r11d, %edx
 3731        cmpl	$12, %edx
 3732        jne	L_AES_GCM_init_aesni_iv_not_12
 3733        # # Calculate values when IV is 12 bytes
 3734        # Set counter based on IV
 3735        movl	$0x1000000, %ecx
 3736        pinsrq	$0x00, (%r10), %xmm4
 3737        pinsrd	$2, 8(%r10), %xmm4
 3738        pinsrd	$3, %ecx, %xmm4
 3739        # H = Encrypt X(=0) and T = Encrypt counter
 3740        movdqa	%xmm4, %xmm1
 3741        movdqa	(%rdi), %xmm5
 3742        pxor	%xmm5, %xmm1
 3743        movdqa	16(%rdi), %xmm6
 3744        aesenc	%xmm6, %xmm5
 3745        aesenc	%xmm6, %xmm1
 3746        movdqa	32(%rdi), %xmm6
 3747        aesenc	%xmm6, %xmm5
 3748        aesenc	%xmm6, %xmm1
 3749        movdqa	48(%rdi), %xmm6
 3750        aesenc	%xmm6, %xmm5
 3751        aesenc	%xmm6, %xmm1
 3752        movdqa	64(%rdi), %xmm6
 3753        aesenc	%xmm6, %xmm5
 3754        aesenc	%xmm6, %xmm1
 3755        movdqa	80(%rdi), %xmm6
 3756        aesenc	%xmm6, %xmm5
 3757        aesenc	%xmm6, %xmm1
 3758        movdqa	96(%rdi), %xmm6
 3759        aesenc	%xmm6, %xmm5
 3760        aesenc	%xmm6, %xmm1
 3761        movdqa	112(%rdi), %xmm6
 3762        aesenc	%xmm6, %xmm5
 3763        aesenc	%xmm6, %xmm1
 3764        movdqa	128(%rdi), %xmm6
 3765        aesenc	%xmm6, %xmm5
 3766        aesenc	%xmm6, %xmm1
 3767        movdqa	144(%rdi), %xmm6
 3768        aesenc	%xmm6, %xmm5
 3769        aesenc	%xmm6, %xmm1
 3770        cmpl	$11, %esi
 3771        movdqa	160(%rdi), %xmm6
 3772        jl	L_AES_GCM_init_aesni_calc_iv_12_last
 3773        aesenc	%xmm6, %xmm5
 3774        aesenc	%xmm6, %xmm1
 3775        movdqa	176(%rdi), %xmm6
 3776        aesenc	%xmm6, %xmm5
 3777        aesenc	%xmm6, %xmm1
 3778        cmpl	$13, %esi
 3779        movdqa	192(%rdi), %xmm6
 3780        jl	L_AES_GCM_init_aesni_calc_iv_12_last
 3781        aesenc	%xmm6, %xmm5
 3782        aesenc	%xmm6, %xmm1
 3783        movdqa	208(%rdi), %xmm6
 3784        aesenc	%xmm6, %xmm5
 3785        aesenc	%xmm6, %xmm1
 3786        movdqa	224(%rdi), %xmm6
 3787L_AES_GCM_init_aesni_calc_iv_12_last:
 3788        aesenclast	%xmm6, %xmm5
 3789        aesenclast	%xmm6, %xmm1
 3790        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm5
 3791        movdqu	%xmm1, %xmm15
 3792        jmp	L_AES_GCM_init_aesni_iv_done
 3793L_AES_GCM_init_aesni_iv_not_12:
 3794        # Calculate values when IV is not 12 bytes
 3795        # H = Encrypt X(=0)
 3796        movdqa	(%rdi), %xmm5
 3797        aesenc	16(%rdi), %xmm5
 3798        aesenc	32(%rdi), %xmm5
 3799        aesenc	48(%rdi), %xmm5
 3800        aesenc	64(%rdi), %xmm5
 3801        aesenc	80(%rdi), %xmm5
 3802        aesenc	96(%rdi), %xmm5
 3803        aesenc	112(%rdi), %xmm5
 3804        aesenc	128(%rdi), %xmm5
 3805        aesenc	144(%rdi), %xmm5
 3806        cmpl	$11, %esi
 3807        movdqa	160(%rdi), %xmm8
 3808        jl	L_AES_GCM_init_aesni_calc_iv_1_aesenc_avx_last
 3809        aesenc	%xmm8, %xmm5
 3810        aesenc	176(%rdi), %xmm5
 3811        cmpl	$13, %esi
 3812        movdqa	192(%rdi), %xmm8
 3813        jl	L_AES_GCM_init_aesni_calc_iv_1_aesenc_avx_last
 3814        aesenc	%xmm8, %xmm5
 3815        aesenc	208(%rdi), %xmm5
 3816        movdqa	224(%rdi), %xmm8
 3817L_AES_GCM_init_aesni_calc_iv_1_aesenc_avx_last:
 3818        aesenclast	%xmm8, %xmm5
 3819        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm5
 3820        # Calc counter
 3821        # Initialization vector
 3822        cmpl	$0x00, %edx
 3823        movq	$0x00, %rcx
 3824        je	L_AES_GCM_init_aesni_calc_iv_done
 3825        cmpl	$16, %edx
 3826        jl	L_AES_GCM_init_aesni_calc_iv_lt16
 3827        andl	$0xfffffff0, %edx
 3828L_AES_GCM_init_aesni_calc_iv_16_loop:
 3829        movdqu	(%r10,%rcx,1), %xmm7
 3830        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm7
 3831        pxor	%xmm7, %xmm4
 3832        pshufd	$0x4e, %xmm4, %xmm1
 3833        pshufd	$0x4e, %xmm5, %xmm2
 3834        movdqa	%xmm5, %xmm3
 3835        movdqa	%xmm5, %xmm0
 3836        pclmulqdq	$0x11, %xmm4, %xmm3
 3837        pclmulqdq	$0x00, %xmm4, %xmm0
 3838        pxor	%xmm4, %xmm1
 3839        pxor	%xmm5, %xmm2
 3840        pclmulqdq	$0x00, %xmm2, %xmm1
 3841        pxor	%xmm0, %xmm1
 3842        pxor	%xmm3, %xmm1
 3843        movdqa	%xmm1, %xmm2
 3844        movdqa	%xmm0, %xmm6
 3845        movdqa	%xmm3, %xmm4
 3846        pslldq	$8, %xmm2
 3847        psrldq	$8, %xmm1
 3848        pxor	%xmm2, %xmm6
 3849        pxor	%xmm1, %xmm4
 3850        movdqa	%xmm6, %xmm0
 3851        movdqa	%xmm4, %xmm1
 3852        psrld	$31, %xmm0
 3853        psrld	$31, %xmm1
 3854        pslld	$0x01, %xmm6
 3855        pslld	$0x01, %xmm4
 3856        movdqa	%xmm0, %xmm2
 3857        pslldq	$4, %xmm0
 3858        psrldq	$12, %xmm2
 3859        pslldq	$4, %xmm1
 3860        por	%xmm2, %xmm4
 3861        por	%xmm0, %xmm6
 3862        por	%xmm1, %xmm4
 3863        movdqa	%xmm6, %xmm0
 3864        movdqa	%xmm6, %xmm1
 3865        movdqa	%xmm6, %xmm2
 3866        pslld	$31, %xmm0
 3867        pslld	$30, %xmm1
 3868        pslld	$25, %xmm2
 3869        pxor	%xmm1, %xmm0
 3870        pxor	%xmm2, %xmm0
 3871        movdqa	%xmm0, %xmm1
 3872        psrldq	$4, %xmm1
 3873        pslldq	$12, %xmm0
 3874        pxor	%xmm0, %xmm6
 3875        movdqa	%xmm6, %xmm2
 3876        movdqa	%xmm6, %xmm3
 3877        movdqa	%xmm6, %xmm0
 3878        psrld	$0x01, %xmm2
 3879        psrld	$2, %xmm3
 3880        psrld	$7, %xmm0
 3881        pxor	%xmm3, %xmm2
 3882        pxor	%xmm0, %xmm2
 3883        pxor	%xmm1, %xmm2
 3884        pxor	%xmm6, %xmm2
 3885        pxor	%xmm2, %xmm4
 3886        addl	$16, %ecx
 3887        cmpl	%edx, %ecx
 3888        jl	L_AES_GCM_init_aesni_calc_iv_16_loop
 3889        movl	%r11d, %edx
 3890        cmpl	%edx, %ecx
 3891        je	L_AES_GCM_init_aesni_calc_iv_done
 3892L_AES_GCM_init_aesni_calc_iv_lt16:
 3893        subq	$16, %rsp
 3894        pxor	%xmm7, %xmm7
 3895        xorl	%r13d, %r13d
 3896        movdqu	%xmm7, (%rsp)
 3897L_AES_GCM_init_aesni_calc_iv_loop:
 3898        movzbl	(%r10,%rcx,1), %r12d
 3899        movb	%r12b, (%rsp,%r13,1)
 3900        incl	%ecx
 3901        incl	%r13d
 3902        cmpl	%edx, %ecx
 3903        jl	L_AES_GCM_init_aesni_calc_iv_loop
 3904        movdqu	(%rsp), %xmm7
 3905        addq	$16, %rsp
 3906        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm7
 3907        pxor	%xmm7, %xmm4
 3908        pshufd	$0x4e, %xmm4, %xmm1
 3909        pshufd	$0x4e, %xmm5, %xmm2
 3910        movdqa	%xmm5, %xmm3
 3911        movdqa	%xmm5, %xmm0
 3912        pclmulqdq	$0x11, %xmm4, %xmm3
 3913        pclmulqdq	$0x00, %xmm4, %xmm0
 3914        pxor	%xmm4, %xmm1
 3915        pxor	%xmm5, %xmm2
 3916        pclmulqdq	$0x00, %xmm2, %xmm1
 3917        pxor	%xmm0, %xmm1
 3918        pxor	%xmm3, %xmm1
 3919        movdqa	%xmm1, %xmm2
 3920        movdqa	%xmm0, %xmm6
 3921        movdqa	%xmm3, %xmm4
 3922        pslldq	$8, %xmm2
 3923        psrldq	$8, %xmm1
 3924        pxor	%xmm2, %xmm6
 3925        pxor	%xmm1, %xmm4
 3926        movdqa	%xmm6, %xmm0
 3927        movdqa	%xmm4, %xmm1
 3928        psrld	$31, %xmm0
 3929        psrld	$31, %xmm1
 3930        pslld	$0x01, %xmm6
 3931        pslld	$0x01, %xmm4
 3932        movdqa	%xmm0, %xmm2
 3933        pslldq	$4, %xmm0
 3934        psrldq	$12, %xmm2
 3935        pslldq	$4, %xmm1
 3936        por	%xmm2, %xmm4
 3937        por	%xmm0, %xmm6
 3938        por	%xmm1, %xmm4
 3939        movdqa	%xmm6, %xmm0
 3940        movdqa	%xmm6, %xmm1
 3941        movdqa	%xmm6, %xmm2
 3942        pslld	$31, %xmm0
 3943        pslld	$30, %xmm1
 3944        pslld	$25, %xmm2
 3945        pxor	%xmm1, %xmm0
 3946        pxor	%xmm2, %xmm0
 3947        movdqa	%xmm0, %xmm1
 3948        psrldq	$4, %xmm1
 3949        pslldq	$12, %xmm0
 3950        pxor	%xmm0, %xmm6
 3951        movdqa	%xmm6, %xmm2
 3952        movdqa	%xmm6, %xmm3
 3953        movdqa	%xmm6, %xmm0
 3954        psrld	$0x01, %xmm2
 3955        psrld	$2, %xmm3
 3956        psrld	$7, %xmm0
 3957        pxor	%xmm3, %xmm2
 3958        pxor	%xmm0, %xmm2
 3959        pxor	%xmm1, %xmm2
 3960        pxor	%xmm6, %xmm2
 3961        pxor	%xmm2, %xmm4
 3962L_AES_GCM_init_aesni_calc_iv_done:
 3963        # T = Encrypt counter
 3964        pxor	%xmm0, %xmm0
 3965        shll	$3, %edx
 3966        pinsrq	$0x00, %rdx, %xmm0
 3967        pxor	%xmm0, %xmm4
 3968        pshufd	$0x4e, %xmm4, %xmm1
 3969        pshufd	$0x4e, %xmm5, %xmm2
 3970        movdqa	%xmm5, %xmm3
 3971        movdqa	%xmm5, %xmm0
 3972        pclmulqdq	$0x11, %xmm4, %xmm3
 3973        pclmulqdq	$0x00, %xmm4, %xmm0
 3974        pxor	%xmm4, %xmm1
 3975        pxor	%xmm5, %xmm2
 3976        pclmulqdq	$0x00, %xmm2, %xmm1
 3977        pxor	%xmm0, %xmm1
 3978        pxor	%xmm3, %xmm1
 3979        movdqa	%xmm1, %xmm2
 3980        movdqa	%xmm0, %xmm6
 3981        movdqa	%xmm3, %xmm4
 3982        pslldq	$8, %xmm2
 3983        psrldq	$8, %xmm1
 3984        pxor	%xmm2, %xmm6
 3985        pxor	%xmm1, %xmm4
 3986        movdqa	%xmm6, %xmm0
 3987        movdqa	%xmm4, %xmm1
 3988        psrld	$31, %xmm0
 3989        psrld	$31, %xmm1
 3990        pslld	$0x01, %xmm6
 3991        pslld	$0x01, %xmm4
 3992        movdqa	%xmm0, %xmm2
 3993        pslldq	$4, %xmm0
 3994        psrldq	$12, %xmm2
 3995        pslldq	$4, %xmm1
 3996        por	%xmm2, %xmm4
 3997        por	%xmm0, %xmm6
 3998        por	%xmm1, %xmm4
 3999        movdqa	%xmm6, %xmm0
 4000        movdqa	%xmm6, %xmm1
 4001        movdqa	%xmm6, %xmm2
 4002        pslld	$31, %xmm0
 4003        pslld	$30, %xmm1
 4004        pslld	$25, %xmm2
 4005        pxor	%xmm1, %xmm0
 4006        pxor	%xmm2, %xmm0
 4007        movdqa	%xmm0, %xmm1
 4008        psrldq	$4, %xmm1
 4009        pslldq	$12, %xmm0
 4010        pxor	%xmm0, %xmm6
 4011        movdqa	%xmm6, %xmm2
 4012        movdqa	%xmm6, %xmm3
 4013        movdqa	%xmm6, %xmm0
 4014        psrld	$0x01, %xmm2
 4015        psrld	$2, %xmm3
 4016        psrld	$7, %xmm0
 4017        pxor	%xmm3, %xmm2
 4018        pxor	%xmm0, %xmm2
 4019        pxor	%xmm1, %xmm2
 4020        pxor	%xmm6, %xmm2
 4021        pxor	%xmm2, %xmm4
 4022        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm4
 4023        #   Encrypt counter
 4024        movdqa	(%rdi), %xmm7
 4025        pxor	%xmm4, %xmm7
 4026        aesenc	16(%rdi), %xmm7
 4027        aesenc	32(%rdi), %xmm7
 4028        aesenc	48(%rdi), %xmm7
 4029        aesenc	64(%rdi), %xmm7
 4030        aesenc	80(%rdi), %xmm7
 4031        aesenc	96(%rdi), %xmm7
 4032        aesenc	112(%rdi), %xmm7
 4033        aesenc	128(%rdi), %xmm7
 4034        aesenc	144(%rdi), %xmm7
 4035        cmpl	$11, %esi
 4036        movdqa	160(%rdi), %xmm8
 4037        jl	L_AES_GCM_init_aesni_calc_iv_2_aesenc_avx_last
 4038        aesenc	%xmm8, %xmm7
 4039        aesenc	176(%rdi), %xmm7
 4040        cmpl	$13, %esi
 4041        movdqa	192(%rdi), %xmm8
 4042        jl	L_AES_GCM_init_aesni_calc_iv_2_aesenc_avx_last
 4043        aesenc	%xmm8, %xmm7
 4044        aesenc	208(%rdi), %xmm7
 4045        movdqa	224(%rdi), %xmm8
 4046L_AES_GCM_init_aesni_calc_iv_2_aesenc_avx_last:
 4047        aesenclast	%xmm8, %xmm7
 4048        movdqu	%xmm7, %xmm15
 4049L_AES_GCM_init_aesni_iv_done:
 4050        movdqa	%xmm15, (%rax)
 4051        pshufb	L_aes_gcm_bswap_epi64(%rip), %xmm4
 4052        paddd	L_aes_gcm_one(%rip), %xmm4
 4053        movdqa	%xmm5, (%r8)
 4054        movdqa	%xmm4, (%r9)
 4055        addq	$16, %rsp
 4056        popq	%r14
 4057        popq	%r13
 4058        popq	%r12
 4059        repz retq
 4060#ifndef __APPLE__
 4061.size	AES_GCM_init_aesni,.-AES_GCM_init_aesni
 4062#endif /* __APPLE__ */
 4063#ifndef __APPLE__
 4064.text
 4065.globl	AES_GCM_aad_update_aesni
 4066.type	AES_GCM_aad_update_aesni,@function
 4067.align	16
 4068AES_GCM_aad_update_aesni:
 4069#else
 4070.section	__TEXT,__text
 4071.globl	_AES_GCM_aad_update_aesni
 4072.p2align	4
 4073_AES_GCM_aad_update_aesni:
 4074#endif /* __APPLE__ */
 4075        movq	%rcx, %rax
 4076        movdqa	(%rdx), %xmm5
 4077        movdqa	(%rax), %xmm6
 4078        xorl	%ecx, %ecx
 4079L_AES_GCM_aad_update_aesni_16_loop:
 4080        movdqu	(%rdi,%rcx,1), %xmm7
 4081        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm7
 4082        pxor	%xmm7, %xmm5
 4083        pshufd	$0x4e, %xmm5, %xmm1
 4084        pshufd	$0x4e, %xmm6, %xmm2
 4085        movdqa	%xmm6, %xmm3
 4086        movdqa	%xmm6, %xmm0
 4087        pclmulqdq	$0x11, %xmm5, %xmm3
 4088        pclmulqdq	$0x00, %xmm5, %xmm0
 4089        pxor	%xmm5, %xmm1
 4090        pxor	%xmm6, %xmm2
 4091        pclmulqdq	$0x00, %xmm2, %xmm1
 4092        pxor	%xmm0, %xmm1
 4093        pxor	%xmm3, %xmm1
 4094        movdqa	%xmm1, %xmm2
 4095        movdqa	%xmm0, %xmm4
 4096        movdqa	%xmm3, %xmm5
 4097        pslldq	$8, %xmm2
 4098        psrldq	$8, %xmm1
 4099        pxor	%xmm2, %xmm4
 4100        pxor	%xmm1, %xmm5
 4101        movdqa	%xmm4, %xmm0
 4102        movdqa	%xmm5, %xmm1
 4103        psrld	$31, %xmm0
 4104        psrld	$31, %xmm1
 4105        pslld	$0x01, %xmm4
 4106        pslld	$0x01, %xmm5
 4107        movdqa	%xmm0, %xmm2
 4108        pslldq	$4, %xmm0
 4109        psrldq	$12, %xmm2
 4110        pslldq	$4, %xmm1
 4111        por	%xmm2, %xmm5
 4112        por	%xmm0, %xmm4
 4113        por	%xmm1, %xmm5
 4114        movdqa	%xmm4, %xmm0
 4115        movdqa	%xmm4, %xmm1
 4116        movdqa	%xmm4, %xmm2
 4117        pslld	$31, %xmm0
 4118        pslld	$30, %xmm1
 4119        pslld	$25, %xmm2
 4120        pxor	%xmm1, %xmm0
 4121        pxor	%xmm2, %xmm0
 4122        movdqa	%xmm0, %xmm1
 4123        psrldq	$4, %xmm1
 4124        pslldq	$12, %xmm0
 4125        pxor	%xmm0, %xmm4
 4126        movdqa	%xmm4, %xmm2
 4127        movdqa	%xmm4, %xmm3
 4128        movdqa	%xmm4, %xmm0
 4129        psrld	$0x01, %xmm2
 4130        psrld	$2, %xmm3
 4131        psrld	$7, %xmm0
 4132        pxor	%xmm3, %xmm2
 4133        pxor	%xmm0, %xmm2
 4134        pxor	%xmm1, %xmm2
 4135        pxor	%xmm4, %xmm2
 4136        pxor	%xmm2, %xmm5
 4137        addl	$16, %ecx
 4138        cmpl	%esi, %ecx
 4139        jl	L_AES_GCM_aad_update_aesni_16_loop
 4140        movdqa	%xmm5, (%rdx)
 4141        repz retq
 4142#ifndef __APPLE__
 4143.size	AES_GCM_aad_update_aesni,.-AES_GCM_aad_update_aesni
 4144#endif /* __APPLE__ */
 4145#ifndef __APPLE__
 4146.text
 4147.globl	AES_GCM_encrypt_block_aesni
 4148.type	AES_GCM_encrypt_block_aesni,@function
 4149.align	16
 4150AES_GCM_encrypt_block_aesni:
 4151#else
 4152.section	__TEXT,__text
 4153.globl	_AES_GCM_encrypt_block_aesni
 4154.p2align	4
 4155_AES_GCM_encrypt_block_aesni:
 4156#endif /* __APPLE__ */
 4157        movq	%rdx, %r10
 4158        movq	%rcx, %r11
 4159        movdqu	(%r8), %xmm0
 4160        movdqa	%xmm0, %xmm1
 4161        pshufb	L_aes_gcm_bswap_epi64(%rip), %xmm0
 4162        paddd	L_aes_gcm_one(%rip), %xmm1
 4163        pxor	(%rdi), %xmm0
 4164        movdqu	%xmm1, (%r8)
 4165        aesenc	16(%rdi), %xmm0
 4166        aesenc	32(%rdi), %xmm0
 4167        aesenc	48(%rdi), %xmm0
 4168        aesenc	64(%rdi), %xmm0
 4169        aesenc	80(%rdi), %xmm0
 4170        aesenc	96(%rdi), %xmm0
 4171        aesenc	112(%rdi), %xmm0
 4172        aesenc	128(%rdi), %xmm0
 4173        aesenc	144(%rdi), %xmm0
 4174        cmpl	$11, %esi
 4175        movdqa	160(%rdi), %xmm1
 4176        jl	L_AES_GCM_encrypt_block_aesni_aesenc_block_aesenc_avx_last
 4177        aesenc	%xmm1, %xmm0
 4178        aesenc	176(%rdi), %xmm0
 4179        cmpl	$13, %esi
 4180        movdqa	192(%rdi), %xmm1
 4181        jl	L_AES_GCM_encrypt_block_aesni_aesenc_block_aesenc_avx_last
 4182        aesenc	%xmm1, %xmm0
 4183        aesenc	208(%rdi), %xmm0
 4184        movdqa	224(%rdi), %xmm1
 4185L_AES_GCM_encrypt_block_aesni_aesenc_block_aesenc_avx_last:
 4186        aesenclast	%xmm1, %xmm0
 4187        movdqu	(%r11), %xmm1
 4188        pxor	%xmm1, %xmm0
 4189        movdqu	%xmm0, (%r10)
 4190        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm0
 4191        repz retq
 4192#ifndef __APPLE__
 4193.size	AES_GCM_encrypt_block_aesni,.-AES_GCM_encrypt_block_aesni
 4194#endif /* __APPLE__ */
 4195#ifndef __APPLE__
 4196.text
 4197.globl	AES_GCM_ghash_block_aesni
 4198.type	AES_GCM_ghash_block_aesni,@function
 4199.align	16
 4200AES_GCM_ghash_block_aesni:
 4201#else
 4202.section	__TEXT,__text
 4203.globl	_AES_GCM_ghash_block_aesni
 4204.p2align	4
 4205_AES_GCM_ghash_block_aesni:
 4206#endif /* __APPLE__ */
 4207        movdqa	(%rsi), %xmm4
 4208        movdqa	(%rdx), %xmm5
 4209        movdqu	(%rdi), %xmm7
 4210        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm7
 4211        pxor	%xmm7, %xmm4
 4212        pshufd	$0x4e, %xmm4, %xmm1
 4213        pshufd	$0x4e, %xmm5, %xmm2
 4214        movdqa	%xmm5, %xmm3
 4215        movdqa	%xmm5, %xmm0
 4216        pclmulqdq	$0x11, %xmm4, %xmm3
 4217        pclmulqdq	$0x00, %xmm4, %xmm0
 4218        pxor	%xmm4, %xmm1
 4219        pxor	%xmm5, %xmm2
 4220        pclmulqdq	$0x00, %xmm2, %xmm1
 4221        pxor	%xmm0, %xmm1
 4222        pxor	%xmm3, %xmm1
 4223        movdqa	%xmm1, %xmm2
 4224        movdqa	%xmm0, %xmm6
 4225        movdqa	%xmm3, %xmm4
 4226        pslldq	$8, %xmm2
 4227        psrldq	$8, %xmm1
 4228        pxor	%xmm2, %xmm6
 4229        pxor	%xmm1, %xmm4
 4230        movdqa	%xmm6, %xmm0
 4231        movdqa	%xmm4, %xmm1
 4232        psrld	$31, %xmm0
 4233        psrld	$31, %xmm1
 4234        pslld	$0x01, %xmm6
 4235        pslld	$0x01, %xmm4
 4236        movdqa	%xmm0, %xmm2
 4237        pslldq	$4, %xmm0
 4238        psrldq	$12, %xmm2
 4239        pslldq	$4, %xmm1
 4240        por	%xmm2, %xmm4
 4241        por	%xmm0, %xmm6
 4242        por	%xmm1, %xmm4
 4243        movdqa	%xmm6, %xmm0
 4244        movdqa	%xmm6, %xmm1
 4245        movdqa	%xmm6, %xmm2
 4246        pslld	$31, %xmm0
 4247        pslld	$30, %xmm1
 4248        pslld	$25, %xmm2
 4249        pxor	%xmm1, %xmm0
 4250        pxor	%xmm2, %xmm0
 4251        movdqa	%xmm0, %xmm1
 4252        psrldq	$4, %xmm1
 4253        pslldq	$12, %xmm0
 4254        pxor	%xmm0, %xmm6
 4255        movdqa	%xmm6, %xmm2
 4256        movdqa	%xmm6, %xmm3
 4257        movdqa	%xmm6, %xmm0
 4258        psrld	$0x01, %xmm2
 4259        psrld	$2, %xmm3
 4260        psrld	$7, %xmm0
 4261        pxor	%xmm3, %xmm2
 4262        pxor	%xmm0, %xmm2
 4263        pxor	%xmm1, %xmm2
 4264        pxor	%xmm6, %xmm2
 4265        pxor	%xmm2, %xmm4
 4266        movdqa	%xmm4, (%rsi)
 4267        repz retq
 4268#ifndef __APPLE__
 4269.size	AES_GCM_ghash_block_aesni,.-AES_GCM_ghash_block_aesni
 4270#endif /* __APPLE__ */
 4271#ifndef __APPLE__
 4272.text
 4273.globl	AES_GCM_encrypt_update_aesni
 4274.type	AES_GCM_encrypt_update_aesni,@function
 4275.align	16
 4276AES_GCM_encrypt_update_aesni:
 4277#else
 4278.section	__TEXT,__text
 4279.globl	_AES_GCM_encrypt_update_aesni
 4280.p2align	4
 4281_AES_GCM_encrypt_update_aesni:
 4282#endif /* __APPLE__ */
 4283        pushq	%r13
 4284        pushq	%r12
 4285        pushq	%r14
 4286        movq	%rdx, %r10
 4287        movq	%rcx, %r11
 4288        movq	32(%rsp), %rax
 4289        movq	40(%rsp), %r12
 4290        subq	$0xa0, %rsp
 4291        movdqa	(%r9), %xmm6
 4292        movdqa	(%rax), %xmm5
 4293        movdqa	%xmm5, %xmm9
 4294        movdqa	%xmm5, %xmm8
 4295        psrlq	$63, %xmm9
 4296        psllq	$0x01, %xmm8
 4297        pslldq	$8, %xmm9
 4298        por	%xmm9, %xmm8
 4299        pshufd	$0xff, %xmm5, %xmm5
 4300        psrad	$31, %xmm5
 4301        pand	L_aes_gcm_mod2_128(%rip), %xmm5
 4302        pxor	%xmm8, %xmm5
 4303        xorq	%r14, %r14
 4304        cmpl	$0x80, %r8d
 4305        movl	%r8d, %r13d
 4306        jl	L_AES_GCM_encrypt_update_aesni_done_128
 4307        andl	$0xffffff80, %r13d
 4308        movdqa	%xmm6, %xmm2
 4309        # H ^ 1
 4310        movdqu	%xmm5, (%rsp)
 4311        # H ^ 2
 4312        pshufd	$0x4e, %xmm5, %xmm9
 4313        pshufd	$0x4e, %xmm5, %xmm10
 4314        movdqa	%xmm5, %xmm11
 4315        movdqa	%xmm5, %xmm8
 4316        pclmulqdq	$0x11, %xmm5, %xmm11
 4317        pclmulqdq	$0x00, %xmm5, %xmm8
 4318        pxor	%xmm5, %xmm9
 4319        pxor	%xmm5, %xmm10
 4320        pclmulqdq	$0x00, %xmm10, %xmm9
 4321        pxor	%xmm8, %xmm9
 4322        pxor	%xmm11, %xmm9
 4323        movdqa	%xmm9, %xmm10
 4324        movdqa	%xmm11, %xmm0
 4325        pslldq	$8, %xmm10
 4326        psrldq	$8, %xmm9
 4327        pxor	%xmm10, %xmm8
 4328        pxor	%xmm9, %xmm0
 4329        movdqa	%xmm8, %xmm12
 4330        movdqa	%xmm8, %xmm13
 4331        movdqa	%xmm8, %xmm14
 4332        pslld	$31, %xmm12
 4333        pslld	$30, %xmm13
 4334        pslld	$25, %xmm14
 4335        pxor	%xmm13, %xmm12
 4336        pxor	%xmm14, %xmm12
 4337        movdqa	%xmm12, %xmm13
 4338        psrldq	$4, %xmm13
 4339        pslldq	$12, %xmm12
 4340        pxor	%xmm12, %xmm8
 4341        movdqa	%xmm8, %xmm14
 4342        movdqa	%xmm8, %xmm10
 4343        movdqa	%xmm8, %xmm9
 4344        psrld	$0x01, %xmm14
 4345        psrld	$2, %xmm10
 4346        psrld	$7, %xmm9
 4347        pxor	%xmm10, %xmm14
 4348        pxor	%xmm9, %xmm14
 4349        pxor	%xmm13, %xmm14
 4350        pxor	%xmm8, %xmm14
 4351        pxor	%xmm14, %xmm0
 4352        movdqu	%xmm0, 16(%rsp)
 4353        # H ^ 3
 4354        pshufd	$0x4e, %xmm5, %xmm9
 4355        pshufd	$0x4e, %xmm0, %xmm10
 4356        movdqa	%xmm0, %xmm11
 4357        movdqa	%xmm0, %xmm8
 4358        pclmulqdq	$0x11, %xmm5, %xmm11
 4359        pclmulqdq	$0x00, %xmm5, %xmm8
 4360        pxor	%xmm5, %xmm9
 4361        pxor	%xmm0, %xmm10
 4362        pclmulqdq	$0x00, %xmm10, %xmm9
 4363        pxor	%xmm8, %xmm9
 4364        pxor	%xmm11, %xmm9
 4365        movdqa	%xmm9, %xmm10
 4366        movdqa	%xmm11, %xmm1
 4367        pslldq	$8, %xmm10
 4368        psrldq	$8, %xmm9
 4369        pxor	%xmm10, %xmm8
 4370        pxor	%xmm9, %xmm1
 4371        movdqa	%xmm8, %xmm12
 4372        movdqa	%xmm8, %xmm13
 4373        movdqa	%xmm8, %xmm14
 4374        pslld	$31, %xmm12
 4375        pslld	$30, %xmm13
 4376        pslld	$25, %xmm14
 4377        pxor	%xmm13, %xmm12
 4378        pxor	%xmm14, %xmm12
 4379        movdqa	%xmm12, %xmm13
 4380        psrldq	$4, %xmm13
 4381        pslldq	$12, %xmm12
 4382        pxor	%xmm12, %xmm8
 4383        movdqa	%xmm8, %xmm14
 4384        movdqa	%xmm8, %xmm10
 4385        movdqa	%xmm8, %xmm9
 4386        psrld	$0x01, %xmm14
 4387        psrld	$2, %xmm10
 4388        psrld	$7, %xmm9
 4389        pxor	%xmm10, %xmm14
 4390        pxor	%xmm9, %xmm14
 4391        pxor	%xmm13, %xmm14
 4392        pxor	%xmm8, %xmm14
 4393        pxor	%xmm14, %xmm1
 4394        movdqu	%xmm1, 32(%rsp)
 4395        # H ^ 4
 4396        pshufd	$0x4e, %xmm0, %xmm9
 4397        pshufd	$0x4e, %xmm0, %xmm10
 4398        movdqa	%xmm0, %xmm11
 4399        movdqa	%xmm0, %xmm8
 4400        pclmulqdq	$0x11, %xmm0, %xmm11
 4401        pclmulqdq	$0x00, %xmm0, %xmm8
 4402        pxor	%xmm0, %xmm9
 4403        pxor	%xmm0, %xmm10
 4404        pclmulqdq	$0x00, %xmm10, %xmm9
 4405        pxor	%xmm8, %xmm9
 4406        pxor	%xmm11, %xmm9
 4407        movdqa	%xmm9, %xmm10
 4408        movdqa	%xmm11, %xmm3
 4409        pslldq	$8, %xmm10
 4410        psrldq	$8, %xmm9
 4411        pxor	%xmm10, %xmm8
 4412        pxor	%xmm9, %xmm3
 4413        movdqa	%xmm8, %xmm12
 4414        movdqa	%xmm8, %xmm13
 4415        movdqa	%xmm8, %xmm14
 4416        pslld	$31, %xmm12
 4417        pslld	$30, %xmm13
 4418        pslld	$25, %xmm14
 4419        pxor	%xmm13, %xmm12
 4420        pxor	%xmm14, %xmm12
 4421        movdqa	%xmm12, %xmm13
 4422        psrldq	$4, %xmm13
 4423        pslldq	$12, %xmm12
 4424        pxor	%xmm12, %xmm8
 4425        movdqa	%xmm8, %xmm14
 4426        movdqa	%xmm8, %xmm10
 4427        movdqa	%xmm8, %xmm9
 4428        psrld	$0x01, %xmm14
 4429        psrld	$2, %xmm10
 4430        psrld	$7, %xmm9
 4431        pxor	%xmm10, %xmm14
 4432        pxor	%xmm9, %xmm14
 4433        pxor	%xmm13, %xmm14
 4434        pxor	%xmm8, %xmm14
 4435        pxor	%xmm14, %xmm3
 4436        movdqu	%xmm3, 48(%rsp)
 4437        # H ^ 5
 4438        pshufd	$0x4e, %xmm0, %xmm9
 4439        pshufd	$0x4e, %xmm1, %xmm10
 4440        movdqa	%xmm1, %xmm11
 4441        movdqa	%xmm1, %xmm8
 4442        pclmulqdq	$0x11, %xmm0, %xmm11
 4443        pclmulqdq	$0x00, %xmm0, %xmm8
 4444        pxor	%xmm0, %xmm9
 4445        pxor	%xmm1, %xmm10
 4446        pclmulqdq	$0x00, %xmm10, %xmm9
 4447        pxor	%xmm8, %xmm9
 4448        pxor	%xmm11, %xmm9
 4449        movdqa	%xmm9, %xmm10
 4450        movdqa	%xmm11, %xmm7
 4451        pslldq	$8, %xmm10
 4452        psrldq	$8, %xmm9
 4453        pxor	%xmm10, %xmm8
 4454        pxor	%xmm9, %xmm7
 4455        movdqa	%xmm8, %xmm12
 4456        movdqa	%xmm8, %xmm13
 4457        movdqa	%xmm8, %xmm14
 4458        pslld	$31, %xmm12
 4459        pslld	$30, %xmm13
 4460        pslld	$25, %xmm14
 4461        pxor	%xmm13, %xmm12
 4462        pxor	%xmm14, %xmm12
 4463        movdqa	%xmm12, %xmm13
 4464        psrldq	$4, %xmm13
 4465        pslldq	$12, %xmm12
 4466        pxor	%xmm12, %xmm8
 4467        movdqa	%xmm8, %xmm14
 4468        movdqa	%xmm8, %xmm10
 4469        movdqa	%xmm8, %xmm9
 4470        psrld	$0x01, %xmm14
 4471        psrld	$2, %xmm10
 4472        psrld	$7, %xmm9
 4473        pxor	%xmm10, %xmm14
 4474        pxor	%xmm9, %xmm14
 4475        pxor	%xmm13, %xmm14
 4476        pxor	%xmm8, %xmm14
 4477        pxor	%xmm14, %xmm7
 4478        movdqu	%xmm7, 64(%rsp)
 4479        # H ^ 6
 4480        pshufd	$0x4e, %xmm1, %xmm9
 4481        pshufd	$0x4e, %xmm1, %xmm10
 4482        movdqa	%xmm1, %xmm11
 4483        movdqa	%xmm1, %xmm8
 4484        pclmulqdq	$0x11, %xmm1, %xmm11
 4485        pclmulqdq	$0x00, %xmm1, %xmm8
 4486        pxor	%xmm1, %xmm9
 4487        pxor	%xmm1, %xmm10
 4488        pclmulqdq	$0x00, %xmm10, %xmm9
 4489        pxor	%xmm8, %xmm9
 4490        pxor	%xmm11, %xmm9
 4491        movdqa	%xmm9, %xmm10
 4492        movdqa	%xmm11, %xmm7
 4493        pslldq	$8, %xmm10
 4494        psrldq	$8, %xmm9
 4495        pxor	%xmm10, %xmm8
 4496        pxor	%xmm9, %xmm7
 4497        movdqa	%xmm8, %xmm12
 4498        movdqa	%xmm8, %xmm13
 4499        movdqa	%xmm8, %xmm14
 4500        pslld	$31, %xmm12
 4501        pslld	$30, %xmm13
 4502        pslld	$25, %xmm14
 4503        pxor	%xmm13, %xmm12
 4504        pxor	%xmm14, %xmm12
 4505        movdqa	%xmm12, %xmm13
 4506        psrldq	$4, %xmm13
 4507        pslldq	$12, %xmm12
 4508        pxor	%xmm12, %xmm8
 4509        movdqa	%xmm8, %xmm14
 4510        movdqa	%xmm8, %xmm10
 4511        movdqa	%xmm8, %xmm9
 4512        psrld	$0x01, %xmm14
 4513        psrld	$2, %xmm10
 4514        psrld	$7, %xmm9
 4515        pxor	%xmm10, %xmm14
 4516        pxor	%xmm9, %xmm14
 4517        pxor	%xmm13, %xmm14
 4518        pxor	%xmm8, %xmm14
 4519        pxor	%xmm14, %xmm7
 4520        movdqu	%xmm7, 80(%rsp)
 4521        # H ^ 7
 4522        pshufd	$0x4e, %xmm1, %xmm9
 4523        pshufd	$0x4e, %xmm3, %xmm10
 4524        movdqa	%xmm3, %xmm11
 4525        movdqa	%xmm3, %xmm8
 4526        pclmulqdq	$0x11, %xmm1, %xmm11
 4527        pclmulqdq	$0x00, %xmm1, %xmm8
 4528        pxor	%xmm1, %xmm9
 4529        pxor	%xmm3, %xmm10
 4530        pclmulqdq	$0x00, %xmm10, %xmm9
 4531        pxor	%xmm8, %xmm9
 4532        pxor	%xmm11, %xmm9
 4533        movdqa	%xmm9, %xmm10
 4534        movdqa	%xmm11, %xmm7
 4535        pslldq	$8, %xmm10
 4536        psrldq	$8, %xmm9
 4537        pxor	%xmm10, %xmm8
 4538        pxor	%xmm9, %xmm7
 4539        movdqa	%xmm8, %xmm12
 4540        movdqa	%xmm8, %xmm13
 4541        movdqa	%xmm8, %xmm14
 4542        pslld	$31, %xmm12
 4543        pslld	$30, %xmm13
 4544        pslld	$25, %xmm14
 4545        pxor	%xmm13, %xmm12
 4546        pxor	%xmm14, %xmm12
 4547        movdqa	%xmm12, %xmm13
 4548        psrldq	$4, %xmm13
 4549        pslldq	$12, %xmm12
 4550        pxor	%xmm12, %xmm8
 4551        movdqa	%xmm8, %xmm14
 4552        movdqa	%xmm8, %xmm10
 4553        movdqa	%xmm8, %xmm9
 4554        psrld	$0x01, %xmm14
 4555        psrld	$2, %xmm10
 4556        psrld	$7, %xmm9
 4557        pxor	%xmm10, %xmm14
 4558        pxor	%xmm9, %xmm14
 4559        pxor	%xmm13, %xmm14
 4560        pxor	%xmm8, %xmm14
 4561        pxor	%xmm14, %xmm7
 4562        movdqu	%xmm7, 96(%rsp)
 4563        # H ^ 8
 4564        pshufd	$0x4e, %xmm3, %xmm9
 4565        pshufd	$0x4e, %xmm3, %xmm10
 4566        movdqa	%xmm3, %xmm11
 4567        movdqa	%xmm3, %xmm8
 4568        pclmulqdq	$0x11, %xmm3, %xmm11
 4569        pclmulqdq	$0x00, %xmm3, %xmm8
 4570        pxor	%xmm3, %xmm9
 4571        pxor	%xmm3, %xmm10
 4572        pclmulqdq	$0x00, %xmm10, %xmm9
 4573        pxor	%xmm8, %xmm9
 4574        pxor	%xmm11, %xmm9
 4575        movdqa	%xmm9, %xmm10
 4576        movdqa	%xmm11, %xmm7
 4577        pslldq	$8, %xmm10
 4578        psrldq	$8, %xmm9
 4579        pxor	%xmm10, %xmm8
 4580        pxor	%xmm9, %xmm7
 4581        movdqa	%xmm8, %xmm12
 4582        movdqa	%xmm8, %xmm13
 4583        movdqa	%xmm8, %xmm14
 4584        pslld	$31, %xmm12
 4585        pslld	$30, %xmm13
 4586        pslld	$25, %xmm14
 4587        pxor	%xmm13, %xmm12
 4588        pxor	%xmm14, %xmm12
 4589        movdqa	%xmm12, %xmm13
 4590        psrldq	$4, %xmm13
 4591        pslldq	$12, %xmm12
 4592        pxor	%xmm12, %xmm8
 4593        movdqa	%xmm8, %xmm14
 4594        movdqa	%xmm8, %xmm10
 4595        movdqa	%xmm8, %xmm9
 4596        psrld	$0x01, %xmm14
 4597        psrld	$2, %xmm10
 4598        psrld	$7, %xmm9
 4599        pxor	%xmm10, %xmm14
 4600        pxor	%xmm9, %xmm14
 4601        pxor	%xmm13, %xmm14
 4602        pxor	%xmm8, %xmm14
 4603        pxor	%xmm14, %xmm7
 4604        movdqu	%xmm7, 112(%rsp)
 4605        # First 128 bytes of input
 4606        movdqu	(%r12), %xmm8
 4607        movdqa	L_aes_gcm_bswap_epi64(%rip), %xmm1
 4608        movdqa	%xmm8, %xmm0
 4609        pshufb	%xmm1, %xmm8
 4610        movdqa	%xmm0, %xmm9
 4611        paddd	L_aes_gcm_one(%rip), %xmm9
 4612        pshufb	%xmm1, %xmm9
 4613        movdqa	%xmm0, %xmm10
 4614        paddd	L_aes_gcm_two(%rip), %xmm10
 4615        pshufb	%xmm1, %xmm10
 4616        movdqa	%xmm0, %xmm11
 4617        paddd	L_aes_gcm_three(%rip), %xmm11
 4618        pshufb	%xmm1, %xmm11
 4619        movdqa	%xmm0, %xmm12
 4620        paddd	L_aes_gcm_four(%rip), %xmm12
 4621        pshufb	%xmm1, %xmm12
 4622        movdqa	%xmm0, %xmm13
 4623        paddd	L_aes_gcm_five(%rip), %xmm13
 4624        pshufb	%xmm1, %xmm13
 4625        movdqa	%xmm0, %xmm14
 4626        paddd	L_aes_gcm_six(%rip), %xmm14
 4627        pshufb	%xmm1, %xmm14
 4628        movdqa	%xmm0, %xmm15
 4629        paddd	L_aes_gcm_seven(%rip), %xmm15
 4630        pshufb	%xmm1, %xmm15
 4631        paddd	L_aes_gcm_eight(%rip), %xmm0
 4632        movdqa	(%rdi), %xmm7
 4633        movdqu	%xmm0, (%r12)
 4634        pxor	%xmm7, %xmm8
 4635        pxor	%xmm7, %xmm9
 4636        pxor	%xmm7, %xmm10
 4637        pxor	%xmm7, %xmm11
 4638        pxor	%xmm7, %xmm12
 4639        pxor	%xmm7, %xmm13
 4640        pxor	%xmm7, %xmm14
 4641        pxor	%xmm7, %xmm15
 4642        movdqa	16(%rdi), %xmm7
 4643        aesenc	%xmm7, %xmm8
 4644        aesenc	%xmm7, %xmm9
 4645        aesenc	%xmm7, %xmm10
 4646        aesenc	%xmm7, %xmm11
 4647        aesenc	%xmm7, %xmm12
 4648        aesenc	%xmm7, %xmm13
 4649        aesenc	%xmm7, %xmm14
 4650        aesenc	%xmm7, %xmm15
 4651        movdqa	32(%rdi), %xmm7
 4652        aesenc	%xmm7, %xmm8
 4653        aesenc	%xmm7, %xmm9
 4654        aesenc	%xmm7, %xmm10
 4655        aesenc	%xmm7, %xmm11
 4656        aesenc	%xmm7, %xmm12
 4657        aesenc	%xmm7, %xmm13
 4658        aesenc	%xmm7, %xmm14
 4659        aesenc	%xmm7, %xmm15
 4660        movdqa	48(%rdi), %xmm7
 4661        aesenc	%xmm7, %xmm8
 4662        aesenc	%xmm7, %xmm9
 4663        aesenc	%xmm7, %xmm10
 4664        aesenc	%xmm7, %xmm11
 4665        aesenc	%xmm7, %xmm12
 4666        aesenc	%xmm7, %xmm13
 4667        aesenc	%xmm7, %xmm14
 4668        aesenc	%xmm7, %xmm15
 4669        movdqa	64(%rdi), %xmm7
 4670        aesenc	%xmm7, %xmm8
 4671        aesenc	%xmm7, %xmm9
 4672        aesenc	%xmm7, %xmm10
 4673        aesenc	%xmm7, %xmm11
 4674        aesenc	%xmm7, %xmm12
 4675        aesenc	%xmm7, %xmm13
 4676        aesenc	%xmm7, %xmm14
 4677        aesenc	%xmm7, %xmm15
 4678        movdqa	80(%rdi), %xmm7
 4679        aesenc	%xmm7, %xmm8
 4680        aesenc	%xmm7, %xmm9
 4681        aesenc	%xmm7, %xmm10
 4682        aesenc	%xmm7, %xmm11
 4683        aesenc	%xmm7, %xmm12
 4684        aesenc	%xmm7, %xmm13
 4685        aesenc	%xmm7, %xmm14
 4686        aesenc	%xmm7, %xmm15
 4687        movdqa	96(%rdi), %xmm7
 4688        aesenc	%xmm7, %xmm8
 4689        aesenc	%xmm7, %xmm9
 4690        aesenc	%xmm7, %xmm10
 4691        aesenc	%xmm7, %xmm11
 4692        aesenc	%xmm7, %xmm12
 4693        aesenc	%xmm7, %xmm13
 4694        aesenc	%xmm7, %xmm14
 4695        aesenc	%xmm7, %xmm15
 4696        movdqa	112(%rdi), %xmm7
 4697        aesenc	%xmm7, %xmm8
 4698        aesenc	%xmm7, %xmm9
 4699        aesenc	%xmm7, %xmm10
 4700        aesenc	%xmm7, %xmm11
 4701        aesenc	%xmm7, %xmm12
 4702        aesenc	%xmm7, %xmm13
 4703        aesenc	%xmm7, %xmm14
 4704        aesenc	%xmm7, %xmm15
 4705        movdqa	128(%rdi), %xmm7
 4706        aesenc	%xmm7, %xmm8
 4707        aesenc	%xmm7, %xmm9
 4708        aesenc	%xmm7, %xmm10
 4709        aesenc	%xmm7, %xmm11
 4710        aesenc	%xmm7, %xmm12
 4711        aesenc	%xmm7, %xmm13
 4712        aesenc	%xmm7, %xmm14
 4713        aesenc	%xmm7, %xmm15
 4714        movdqa	144(%rdi), %xmm7
 4715        aesenc	%xmm7, %xmm8
 4716        aesenc	%xmm7, %xmm9
 4717        aesenc	%xmm7, %xmm10
 4718        aesenc	%xmm7, %xmm11
 4719        aesenc	%xmm7, %xmm12
 4720        aesenc	%xmm7, %xmm13
 4721        aesenc	%xmm7, %xmm14
 4722        aesenc	%xmm7, %xmm15
 4723        cmpl	$11, %esi
 4724        movdqa	160(%rdi), %xmm7
 4725        jl	L_AES_GCM_encrypt_update_aesni_enc_done
 4726        aesenc	%xmm7, %xmm8
 4727        aesenc	%xmm7, %xmm9
 4728        aesenc	%xmm7, %xmm10
 4729        aesenc	%xmm7, %xmm11
 4730        aesenc	%xmm7, %xmm12
 4731        aesenc	%xmm7, %xmm13
 4732        aesenc	%xmm7, %xmm14
 4733        aesenc	%xmm7, %xmm15
 4734        movdqa	176(%rdi), %xmm7
 4735        aesenc	%xmm7, %xmm8
 4736        aesenc	%xmm7, %xmm9
 4737        aesenc	%xmm7, %xmm10
 4738        aesenc	%xmm7, %xmm11
 4739        aesenc	%xmm7, %xmm12
 4740        aesenc	%xmm7, %xmm13
 4741        aesenc	%xmm7, %xmm14
 4742        aesenc	%xmm7, %xmm15
 4743        cmpl	$13, %esi
 4744        movdqa	192(%rdi), %xmm7
 4745        jl	L_AES_GCM_encrypt_update_aesni_enc_done
 4746        aesenc	%xmm7, %xmm8
 4747        aesenc	%xmm7, %xmm9
 4748        aesenc	%xmm7, %xmm10
 4749        aesenc	%xmm7, %xmm11
 4750        aesenc	%xmm7, %xmm12
 4751        aesenc	%xmm7, %xmm13
 4752        aesenc	%xmm7, %xmm14
 4753        aesenc	%xmm7, %xmm15
 4754        movdqa	208(%rdi), %xmm7
 4755        aesenc	%xmm7, %xmm8
 4756        aesenc	%xmm7, %xmm9
 4757        aesenc	%xmm7, %xmm10
 4758        aesenc	%xmm7, %xmm11
 4759        aesenc	%xmm7, %xmm12
 4760        aesenc	%xmm7, %xmm13
 4761        aesenc	%xmm7, %xmm14
 4762        aesenc	%xmm7, %xmm15
 4763        movdqa	224(%rdi), %xmm7
 4764L_AES_GCM_encrypt_update_aesni_enc_done:
 4765        aesenclast	%xmm7, %xmm8
 4766        aesenclast	%xmm7, %xmm9
 4767        movdqu	(%r11), %xmm0
 4768        movdqu	16(%r11), %xmm1
 4769        pxor	%xmm0, %xmm8
 4770        pxor	%xmm1, %xmm9
 4771        movdqu	%xmm8, (%r10)
 4772        movdqu	%xmm9, 16(%r10)
 4773        aesenclast	%xmm7, %xmm10
 4774        aesenclast	%xmm7, %xmm11
 4775        movdqu	32(%r11), %xmm0
 4776        movdqu	48(%r11), %xmm1
 4777        pxor	%xmm0, %xmm10
 4778        pxor	%xmm1, %xmm11
 4779        movdqu	%xmm10, 32(%r10)
 4780        movdqu	%xmm11, 48(%r10)
 4781        aesenclast	%xmm7, %xmm12
 4782        aesenclast	%xmm7, %xmm13
 4783        movdqu	64(%r11), %xmm0
 4784        movdqu	80(%r11), %xmm1
 4785        pxor	%xmm0, %xmm12
 4786        pxor	%xmm1, %xmm13
 4787        movdqu	%xmm12, 64(%r10)
 4788        movdqu	%xmm13, 80(%r10)
 4789        aesenclast	%xmm7, %xmm14
 4790        aesenclast	%xmm7, %xmm15
 4791        movdqu	96(%r11), %xmm0
 4792        movdqu	112(%r11), %xmm1
 4793        pxor	%xmm0, %xmm14
 4794        pxor	%xmm1, %xmm15
 4795        movdqu	%xmm14, 96(%r10)
 4796        movdqu	%xmm15, 112(%r10)
 4797        cmpl	$0x80, %r13d
 4798        movl	$0x80, %r14d
 4799        jle	L_AES_GCM_encrypt_update_aesni_end_128
 4800        # More 128 bytes of input
 4801L_AES_GCM_encrypt_update_aesni_ghash_128:
 4802        leaq	(%r11,%r14,1), %rcx
 4803        leaq	(%r10,%r14,1), %rdx
 4804        movdqu	(%r12), %xmm8
 4805        movdqa	L_aes_gcm_bswap_epi64(%rip), %xmm1
 4806        movdqa	%xmm8, %xmm0
 4807        pshufb	%xmm1, %xmm8
 4808        movdqa	%xmm0, %xmm9
 4809        paddd	L_aes_gcm_one(%rip), %xmm9
 4810        pshufb	%xmm1, %xmm9
 4811        movdqa	%xmm0, %xmm10
 4812        paddd	L_aes_gcm_two(%rip), %xmm10
 4813        pshufb	%xmm1, %xmm10
 4814        movdqa	%xmm0, %xmm11
 4815        paddd	L_aes_gcm_three(%rip), %xmm11
 4816        pshufb	%xmm1, %xmm11
 4817        movdqa	%xmm0, %xmm12
 4818        paddd	L_aes_gcm_four(%rip), %xmm12
 4819        pshufb	%xmm1, %xmm12
 4820        movdqa	%xmm0, %xmm13
 4821        paddd	L_aes_gcm_five(%rip), %xmm13
 4822        pshufb	%xmm1, %xmm13
 4823        movdqa	%xmm0, %xmm14
 4824        paddd	L_aes_gcm_six(%rip), %xmm14
 4825        pshufb	%xmm1, %xmm14
 4826        movdqa	%xmm0, %xmm15
 4827        paddd	L_aes_gcm_seven(%rip), %xmm15
 4828        pshufb	%xmm1, %xmm15
 4829        paddd	L_aes_gcm_eight(%rip), %xmm0
 4830        movdqa	(%rdi), %xmm7
 4831        movdqu	%xmm0, (%r12)
 4832        pxor	%xmm7, %xmm8
 4833        pxor	%xmm7, %xmm9
 4834        pxor	%xmm7, %xmm10
 4835        pxor	%xmm7, %xmm11
 4836        pxor	%xmm7, %xmm12
 4837        pxor	%xmm7, %xmm13
 4838        pxor	%xmm7, %xmm14
 4839        pxor	%xmm7, %xmm15
 4840        movdqu	112(%rsp), %xmm7
 4841        movdqu	-128(%rdx), %xmm0
 4842        aesenc	16(%rdi), %xmm8
 4843        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm0
 4844        pxor	%xmm2, %xmm0
 4845        pshufd	$0x4e, %xmm7, %xmm1
 4846        pshufd	$0x4e, %xmm0, %xmm5
 4847        pxor	%xmm7, %xmm1
 4848        pxor	%xmm0, %xmm5
 4849        movdqa	%xmm0, %xmm3
 4850        pclmulqdq	$0x11, %xmm7, %xmm3
 4851        aesenc	16(%rdi), %xmm9
 4852        aesenc	16(%rdi), %xmm10
 4853        movdqa	%xmm0, %xmm2
 4854        pclmulqdq	$0x00, %xmm7, %xmm2
 4855        aesenc	16(%rdi), %xmm11
 4856        aesenc	16(%rdi), %xmm12
 4857        pclmulqdq	$0x00, %xmm5, %xmm1
 4858        aesenc	16(%rdi), %xmm13
 4859        aesenc	16(%rdi), %xmm14
 4860        aesenc	16(%rdi), %xmm15
 4861        pxor	%xmm2, %xmm1
 4862        pxor	%xmm3, %xmm1
 4863        movdqu	96(%rsp), %xmm7
 4864        movdqu	-112(%rdx), %xmm0
 4865        pshufd	$0x4e, %xmm7, %xmm4
 4866        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm0
 4867        aesenc	32(%rdi), %xmm8
 4868        pxor	%xmm7, %xmm4
 4869        pshufd	$0x4e, %xmm0, %xmm5
 4870        pxor	%xmm0, %xmm5
 4871        movdqa	%xmm0, %xmm6
 4872        pclmulqdq	$0x11, %xmm7, %xmm6
 4873        aesenc	32(%rdi), %xmm9
 4874        aesenc	32(%rdi), %xmm10
 4875        pclmulqdq	$0x00, %xmm0, %xmm7
 4876        aesenc	32(%rdi), %xmm11
 4877        aesenc	32(%rdi), %xmm12
 4878        pclmulqdq	$0x00, %xmm5, %xmm4
 4879        aesenc	32(%rdi), %xmm13
 4880        aesenc	32(%rdi), %xmm14
 4881        aesenc	32(%rdi), %xmm15
 4882        pxor	%xmm7, %xmm1
 4883        pxor	%xmm7, %xmm2
 4884        pxor	%xmm6, %xmm1
 4885        pxor	%xmm6, %xmm3
 4886        pxor	%xmm4, %xmm1
 4887        movdqu	80(%rsp), %xmm7
 4888        movdqu	-96(%rdx), %xmm0
 4889        pshufd	$0x4e, %xmm7, %xmm4
 4890        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm0
 4891        aesenc	48(%rdi), %xmm8
 4892        pxor	%xmm7, %xmm4
 4893        pshufd	$0x4e, %xmm0, %xmm5
 4894        pxor	%xmm0, %xmm5
 4895        movdqa	%xmm0, %xmm6
 4896        pclmulqdq	$0x11, %xmm7, %xmm6
 4897        aesenc	48(%rdi), %xmm9
 4898        aesenc	48(%rdi), %xmm10
 4899        pclmulqdq	$0x00, %xmm0, %xmm7
 4900        aesenc	48(%rdi), %xmm11
 4901        aesenc	48(%rdi), %xmm12
 4902        pclmulqdq	$0x00, %xmm5, %xmm4
 4903        aesenc	48(%rdi), %xmm13
 4904        aesenc	48(%rdi), %xmm14
 4905        aesenc	48(%rdi), %xmm15
 4906        pxor	%xmm7, %xmm1
 4907        pxor	%xmm7, %xmm2
 4908        pxor	%xmm6, %xmm1
 4909        pxor	%xmm6, %xmm3
 4910        pxor	%xmm4, %xmm1
 4911        movdqu	64(%rsp), %xmm7
 4912        movdqu	-80(%rdx), %xmm0
 4913        pshufd	$0x4e, %xmm7, %xmm4
 4914        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm0
 4915        aesenc	64(%rdi), %xmm8
 4916        pxor	%xmm7, %xmm4
 4917        pshufd	$0x4e, %xmm0, %xmm5
 4918        pxor	%xmm0, %xmm5
 4919        movdqa	%xmm0, %xmm6
 4920        pclmulqdq	$0x11, %xmm7, %xmm6
 4921        aesenc	64(%rdi), %xmm9
 4922        aesenc	64(%rdi), %xmm10
 4923        pclmulqdq	$0x00, %xmm0, %xmm7
 4924        aesenc	64(%rdi), %xmm11
 4925        aesenc	64(%rdi), %xmm12
 4926        pclmulqdq	$0x00, %xmm5, %xmm4
 4927        aesenc	64(%rdi), %xmm13
 4928        aesenc	64(%rdi), %xmm14
 4929        aesenc	64(%rdi), %xmm15
 4930        pxor	%xmm7, %xmm1
 4931        pxor	%xmm7, %xmm2
 4932        pxor	%xmm6, %xmm1
 4933        pxor	%xmm6, %xmm3
 4934        pxor	%xmm4, %xmm1
 4935        movdqu	48(%rsp), %xmm7
 4936        movdqu	-64(%rdx), %xmm0
 4937        pshufd	$0x4e, %xmm7, %xmm4
 4938        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm0
 4939        aesenc	80(%rdi), %xmm8
 4940        pxor	%xmm7, %xmm4
 4941        pshufd	$0x4e, %xmm0, %xmm5
 4942        pxor	%xmm0, %xmm5
 4943        movdqa	%xmm0, %xmm6
 4944        pclmulqdq	$0x11, %xmm7, %xmm6
 4945        aesenc	80(%rdi), %xmm9
 4946        aesenc	80(%rdi), %xmm10
 4947        pclmulqdq	$0x00, %xmm0, %xmm7
 4948        aesenc	80(%rdi), %xmm11
 4949        aesenc	80(%rdi), %xmm12
 4950        pclmulqdq	$0x00, %xmm5, %xmm4
 4951        aesenc	80(%rdi), %xmm13
 4952        aesenc	80(%rdi), %xmm14
 4953        aesenc	80(%rdi), %xmm15
 4954        pxor	%xmm7, %xmm1
 4955        pxor	%xmm7, %xmm2
 4956        pxor	%xmm6, %xmm1
 4957        pxor	%xmm6, %xmm3
 4958        pxor	%xmm4, %xmm1
 4959        movdqu	32(%rsp), %xmm7
 4960        movdqu	-48(%rdx), %xmm0
 4961        pshufd	$0x4e, %xmm7, %xmm4
 4962        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm0
 4963        aesenc	96(%rdi), %xmm8
 4964        pxor	%xmm7, %xmm4
 4965        pshufd	$0x4e, %xmm0, %xmm5
 4966        pxor	%xmm0, %xmm5
 4967        movdqa	%xmm0, %xmm6
 4968        pclmulqdq	$0x11, %xmm7, %xmm6
 4969        aesenc	96(%rdi), %xmm9
 4970        aesenc	96(%rdi), %xmm10
 4971        pclmulqdq	$0x00, %xmm0, %xmm7
 4972        aesenc	96(%rdi), %xmm11
 4973        aesenc	96(%rdi), %xmm12
 4974        pclmulqdq	$0x00, %xmm5, %xmm4
 4975        aesenc	96(%rdi), %xmm13
 4976        aesenc	96(%rdi), %xmm14
 4977        aesenc	96(%rdi), %xmm15
 4978        pxor	%xmm7, %xmm1
 4979        pxor	%xmm7, %xmm2
 4980        pxor	%xmm6, %xmm1
 4981        pxor	%xmm6, %xmm3
 4982        pxor	%xmm4, %xmm1
 4983        movdqu	16(%rsp), %xmm7
 4984        movdqu	-32(%rdx), %xmm0
 4985        pshufd	$0x4e, %xmm7, %xmm4
 4986        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm0
 4987        aesenc	112(%rdi), %xmm8
 4988        pxor	%xmm7, %xmm4
 4989        pshufd	$0x4e, %xmm0, %xmm5
 4990        pxor	%xmm0, %xmm5
 4991        movdqa	%xmm0, %xmm6
 4992        pclmulqdq	$0x11, %xmm7, %xmm6
 4993        aesenc	112(%rdi), %xmm9
 4994        aesenc	112(%rdi), %xmm10
 4995        pclmulqdq	$0x00, %xmm0, %xmm7
 4996        aesenc	112(%rdi), %xmm11
 4997        aesenc	112(%rdi), %xmm12
 4998        pclmulqdq	$0x00, %xmm5, %xmm4
 4999        aesenc	112(%rdi), %xmm13
 5000        aesenc	112(%rdi), %xmm14
 5001        aesenc	112(%rdi), %xmm15
 5002        pxor	%xmm7, %xmm1
 5003        pxor	%xmm7, %xmm2
 5004        pxor	%xmm6, %xmm1
 5005        pxor	%xmm6, %xmm3
 5006        pxor	%xmm4, %xmm1
 5007        movdqu	(%rsp), %xmm7
 5008        movdqu	-16(%rdx), %xmm0
 5009        pshufd	$0x4e, %xmm7, %xmm4
 5010        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm0
 5011        aesenc	128(%rdi), %xmm8
 5012        pxor	%xmm7, %xmm4
 5013        pshufd	$0x4e, %xmm0, %xmm5
 5014        pxor	%xmm0, %xmm5
 5015        movdqa	%xmm0, %xmm6
 5016        pclmulqdq	$0x11, %xmm7, %xmm6
 5017        aesenc	128(%rdi), %xmm9
 5018        aesenc	128(%rdi), %xmm10
 5019        pclmulqdq	$0x00, %xmm0, %xmm7
 5020        aesenc	128(%rdi), %xmm11
 5021        aesenc	128(%rdi), %xmm12
 5022        pclmulqdq	$0x00, %xmm5, %xmm4
 5023        aesenc	128(%rdi), %xmm13
 5024        aesenc	128(%rdi), %xmm14
 5025        aesenc	128(%rdi), %xmm15
 5026        pxor	%xmm7, %xmm1
 5027        pxor	%xmm7, %xmm2
 5028        pxor	%xmm6, %xmm1
 5029        pxor	%xmm6, %xmm3
 5030        pxor	%xmm4, %xmm1
 5031        movdqa	%xmm1, %xmm5
 5032        psrldq	$8, %xmm1
 5033        pslldq	$8, %xmm5
 5034        aesenc	144(%rdi), %xmm8
 5035        pxor	%xmm5, %xmm2
 5036        pxor	%xmm1, %xmm3
 5037        movdqa	%xmm2, %xmm7
 5038        movdqa	%xmm2, %xmm4
 5039        movdqa	%xmm2, %xmm5
 5040        aesenc	144(%rdi), %xmm9
 5041        pslld	$31, %xmm7
 5042        pslld	$30, %xmm4
 5043        pslld	$25, %xmm5
 5044        aesenc	144(%rdi), %xmm10
 5045        pxor	%xmm4, %xmm7
 5046        pxor	%xmm5, %xmm7
 5047        aesenc	144(%rdi), %xmm11
 5048        movdqa	%xmm7, %xmm4
 5049        pslldq	$12, %xmm7
 5050        psrldq	$4, %xmm4
 5051        aesenc	144(%rdi), %xmm12
 5052        pxor	%xmm7, %xmm2
 5053        movdqa	%xmm2, %xmm5
 5054        movdqa	%xmm2, %xmm1
 5055        movdqa	%xmm2, %xmm0
 5056        aesenc	144(%rdi), %xmm13
 5057        psrld	$0x01, %xmm5
 5058        psrld	$2, %xmm1
 5059        psrld	$7, %xmm0
 5060        aesenc	144(%rdi), %xmm14
 5061        pxor	%xmm1, %xmm5
 5062        pxor	%xmm0, %xmm5
 5063        aesenc	144(%rdi), %xmm15
 5064        pxor	%xmm4, %xmm5
 5065        pxor	%xmm5, %xmm2
 5066        pxor	%xmm3, %xmm2
 5067        cmpl	$11, %esi
 5068        movdqa	160(%rdi), %xmm7
 5069        jl	L_AES_GCM_encrypt_update_aesni_aesenc_128_ghash_avx_done
 5070        aesenc	%xmm7, %xmm8
 5071        aesenc	%xmm7, %xmm9
 5072        aesenc	%xmm7, %xmm10
 5073        aesenc	%xmm7, %xmm11
 5074        aesenc	%xmm7, %xmm12
 5075        aesenc	%xmm7, %xmm13
 5076        aesenc	%xmm7, %xmm14
 5077        aesenc	%xmm7, %xmm15
 5078        movdqa	176(%rdi), %xmm7
 5079        aesenc	%xmm7, %xmm8
 5080        aesenc	%xmm7, %xmm9
 5081        aesenc	%xmm7, %xmm10
 5082        aesenc	%xmm7, %xmm11
 5083        aesenc	%xmm7, %xmm12
 5084        aesenc	%xmm7, %xmm13
 5085        aesenc	%xmm7, %xmm14
 5086        aesenc	%xmm7, %xmm15
 5087        cmpl	$13, %esi
 5088        movdqa	192(%rdi), %xmm7
 5089        jl	L_AES_GCM_encrypt_update_aesni_aesenc_128_ghash_avx_done
 5090        aesenc	%xmm7, %xmm8
 5091        aesenc	%xmm7, %xmm9
 5092        aesenc	%xmm7, %xmm10
 5093        aesenc	%xmm7, %xmm11
 5094        aesenc	%xmm7, %xmm12
 5095        aesenc	%xmm7, %xmm13
 5096        aesenc	%xmm7, %xmm14
 5097        aesenc	%xmm7, %xmm15
 5098        movdqa	208(%rdi), %xmm7
 5099        aesenc	%xmm7, %xmm8
 5100        aesenc	%xmm7, %xmm9
 5101        aesenc	%xmm7, %xmm10
 5102        aesenc	%xmm7, %xmm11
 5103        aesenc	%xmm7, %xmm12
 5104        aesenc	%xmm7, %xmm13
 5105        aesenc	%xmm7, %xmm14
 5106        aesenc	%xmm7, %xmm15
 5107        movdqa	224(%rdi), %xmm7
 5108L_AES_GCM_encrypt_update_aesni_aesenc_128_ghash_avx_done:
 5109        aesenclast	%xmm7, %xmm8
 5110        aesenclast	%xmm7, %xmm9
 5111        movdqu	(%rcx), %xmm0
 5112        movdqu	16(%rcx), %xmm1
 5113        pxor	%xmm0, %xmm8
 5114        pxor	%xmm1, %xmm9
 5115        movdqu	%xmm8, (%rdx)
 5116        movdqu	%xmm9, 16(%rdx)
 5117        aesenclast	%xmm7, %xmm10
 5118        aesenclast	%xmm7, %xmm11
 5119        movdqu	32(%rcx), %xmm0
 5120        movdqu	48(%rcx), %xmm1
 5121        pxor	%xmm0, %xmm10
 5122        pxor	%xmm1, %xmm11
 5123        movdqu	%xmm10, 32(%rdx)
 5124        movdqu	%xmm11, 48(%rdx)
 5125        aesenclast	%xmm7, %xmm12
 5126        aesenclast	%xmm7, %xmm13
 5127        movdqu	64(%rcx), %xmm0
 5128        movdqu	80(%rcx), %xmm1
 5129        pxor	%xmm0, %xmm12
 5130        pxor	%xmm1, %xmm13
 5131        movdqu	%xmm12, 64(%rdx)
 5132        movdqu	%xmm13, 80(%rdx)
 5133        aesenclast	%xmm7, %xmm14
 5134        aesenclast	%xmm7, %xmm15
 5135        movdqu	96(%rcx), %xmm0
 5136        movdqu	112(%rcx), %xmm1
 5137        pxor	%xmm0, %xmm14
 5138        pxor	%xmm1, %xmm15
 5139        movdqu	%xmm14, 96(%rdx)
 5140        movdqu	%xmm15, 112(%rdx)
 5141        addl	$0x80, %r14d
 5142        cmpl	%r13d, %r14d
 5143        jl	L_AES_GCM_encrypt_update_aesni_ghash_128
 5144L_AES_GCM_encrypt_update_aesni_end_128:
 5145        movdqa	L_aes_gcm_bswap_mask(%rip), %xmm4
 5146        pshufb	%xmm4, %xmm8
 5147        pshufb	%xmm4, %xmm9
 5148        pshufb	%xmm4, %xmm10
 5149        pshufb	%xmm4, %xmm11
 5150        pxor	%xmm2, %xmm8
 5151        pshufb	%xmm4, %xmm12
 5152        pshufb	%xmm4, %xmm13
 5153        pshufb	%xmm4, %xmm14
 5154        pshufb	%xmm4, %xmm15
 5155        movdqu	112(%rsp), %xmm7
 5156        pshufd	$0x4e, %xmm8, %xmm1
 5157        pshufd	$0x4e, %xmm7, %xmm2
 5158        movdqa	%xmm7, %xmm3
 5159        movdqa	%xmm7, %xmm0
 5160        pclmulqdq	$0x11, %xmm8, %xmm3
 5161        pclmulqdq	$0x00, %xmm8, %xmm0
 5162        pxor	%xmm8, %xmm1
 5163        pxor	%xmm7, %xmm2
 5164        pclmulqdq	$0x00, %xmm2, %xmm1
 5165        pxor	%xmm0, %xmm1
 5166        pxor	%xmm3, %xmm1
 5167        movdqa	%xmm1, %xmm2
 5168        movdqa	%xmm0, %xmm4
 5169        movdqa	%xmm3, %xmm6
 5170        pslldq	$8, %xmm2
 5171        psrldq	$8, %xmm1
 5172        pxor	%xmm2, %xmm4
 5173        pxor	%xmm1, %xmm6
 5174        movdqu	96(%rsp), %xmm7
 5175        pshufd	$0x4e, %xmm9, %xmm1
 5176        pshufd	$0x4e, %xmm7, %xmm2
 5177        movdqa	%xmm7, %xmm3
 5178        movdqa	%xmm7, %xmm0
 5179        pclmulqdq	$0x11, %xmm9, %xmm3
 5180        pclmulqdq	$0x00, %xmm9, %xmm0
 5181        pxor	%xmm9, %xmm1
 5182        pxor	%xmm7, %xmm2
 5183        pclmulqdq	$0x00, %xmm2, %xmm1
 5184        pxor	%xmm0, %xmm1
 5185        pxor	%xmm3, %xmm1
 5186        movdqa	%xmm1, %xmm2
 5187        pxor	%xmm0, %xmm4
 5188        pxor	%xmm3, %xmm6
 5189        pslldq	$8, %xmm2
 5190        psrldq	$8, %xmm1
 5191        pxor	%xmm2, %xmm4
 5192        pxor	%xmm1, %xmm6
 5193        movdqu	80(%rsp), %xmm7
 5194        pshufd	$0x4e, %xmm10, %xmm1
 5195        pshufd	$0x4e, %xmm7, %xmm2
 5196        movdqa	%xmm7, %xmm3
 5197        movdqa	%xmm7, %xmm0
 5198        pclmulqdq	$0x11, %xmm10, %xmm3
 5199        pclmulqdq	$0x00, %xmm10, %xmm0
 5200        pxor	%xmm10, %xmm1
 5201        pxor	%xmm7, %xmm2
 5202        pclmulqdq	$0x00, %xmm2, %xmm1
 5203        pxor	%xmm0, %xmm1
 5204        pxor	%xmm3, %xmm1
 5205        movdqa	%xmm1, %xmm2
 5206        pxor	%xmm0, %xmm4
 5207        pxor	%xmm3, %xmm6
 5208        pslldq	$8, %xmm2
 5209        psrldq	$8, %xmm1
 5210        pxor	%xmm2, %xmm4
 5211        pxor	%xmm1, %xmm6
 5212        movdqu	64(%rsp), %xmm7
 5213        pshufd	$0x4e, %xmm11, %xmm1
 5214        pshufd	$0x4e, %xmm7, %xmm2
 5215        movdqa	%xmm7, %xmm3
 5216        movdqa	%xmm7, %xmm0
 5217        pclmulqdq	$0x11, %xmm11, %xmm3
 5218        pclmulqdq	$0x00, %xmm11, %xmm0
 5219        pxor	%xmm11, %xmm1
 5220        pxor	%xmm7, %xmm2
 5221        pclmulqdq	$0x00, %xmm2, %xmm1
 5222        pxor	%xmm0, %xmm1
 5223        pxor	%xmm3, %xmm1
 5224        movdqa	%xmm1, %xmm2
 5225        pxor	%xmm0, %xmm4
 5226        pxor	%xmm3, %xmm6
 5227        pslldq	$8, %xmm2
 5228        psrldq	$8, %xmm1
 5229        pxor	%xmm2, %xmm4
 5230        pxor	%xmm1, %xmm6
 5231        movdqu	48(%rsp), %xmm7
 5232        pshufd	$0x4e, %xmm12, %xmm1
 5233        pshufd	$0x4e, %xmm7, %xmm2
 5234        movdqa	%xmm7, %xmm3
 5235        movdqa	%xmm7, %xmm0
 5236        pclmulqdq	$0x11, %xmm12, %xmm3
 5237        pclmulqdq	$0x00, %xmm12, %xmm0
 5238        pxor	%xmm12, %xmm1
 5239        pxor	%xmm7, %xmm2
 5240        pclmulqdq	$0x00, %xmm2, %xmm1
 5241        pxor	%xmm0, %xmm1
 5242        pxor	%xmm3, %xmm1
 5243        movdqa	%xmm1, %xmm2
 5244        pxor	%xmm0, %xmm4
 5245        pxor	%xmm3, %xmm6
 5246        pslldq	$8, %xmm2
 5247        psrldq	$8, %xmm1
 5248        pxor	%xmm2, %xmm4
 5249        pxor	%xmm1, %xmm6
 5250        movdqu	32(%rsp), %xmm7
 5251        pshufd	$0x4e, %xmm13, %xmm1
 5252        pshufd	$0x4e, %xmm7, %xmm2
 5253        movdqa	%xmm7, %xmm3
 5254        movdqa	%xmm7, %xmm0
 5255        pclmulqdq	$0x11, %xmm13, %xmm3
 5256        pclmulqdq	$0x00, %xmm13, %xmm0
 5257        pxor	%xmm13, %xmm1
 5258        pxor	%xmm7, %xmm2
 5259        pclmulqdq	$0x00, %xmm2, %xmm1
 5260        pxor	%xmm0, %xmm1
 5261        pxor	%xmm3, %xmm1
 5262        movdqa	%xmm1, %xmm2
 5263        pxor	%xmm0, %xmm4
 5264        pxor	%xmm3, %xmm6
 5265        pslldq	$8, %xmm2
 5266        psrldq	$8, %xmm1
 5267        pxor	%xmm2, %xmm4
 5268        pxor	%xmm1, %xmm6
 5269        movdqu	16(%rsp), %xmm7
 5270        pshufd	$0x4e, %xmm14, %xmm1
 5271        pshufd	$0x4e, %xmm7, %xmm2
 5272        movdqa	%xmm7, %xmm3
 5273        movdqa	%xmm7, %xmm0
 5274        pclmulqdq	$0x11, %xmm14, %xmm3
 5275        pclmulqdq	$0x00, %xmm14, %xmm0
 5276        pxor	%xmm14, %xmm1
 5277        pxor	%xmm7, %xmm2
 5278        pclmulqdq	$0x00, %xmm2, %xmm1
 5279        pxor	%xmm0, %xmm1
 5280        pxor	%xmm3, %xmm1
 5281        movdqa	%xmm1, %xmm2
 5282        pxor	%xmm0, %xmm4
 5283        pxor	%xmm3, %xmm6
 5284        pslldq	$8, %xmm2
 5285        psrldq	$8, %xmm1
 5286        pxor	%xmm2, %xmm4
 5287        pxor	%xmm1, %xmm6
 5288        movdqu	(%rsp), %xmm7
 5289        pshufd	$0x4e, %xmm15, %xmm1
 5290        pshufd	$0x4e, %xmm7, %xmm2
 5291        movdqa	%xmm7, %xmm3
 5292        movdqa	%xmm7, %xmm0
 5293        pclmulqdq	$0x11, %xmm15, %xmm3
 5294        pclmulqdq	$0x00, %xmm15, %xmm0
 5295        pxor	%xmm15, %xmm1
 5296        pxor	%xmm7, %xmm2
 5297        pclmulqdq	$0x00, %xmm2, %xmm1
 5298        pxor	%xmm0, %xmm1
 5299        pxor	%xmm3, %xmm1
 5300        movdqa	%xmm1, %xmm2
 5301        pxor	%xmm0, %xmm4
 5302        pxor	%xmm3, %xmm6
 5303        pslldq	$8, %xmm2
 5304        psrldq	$8, %xmm1
 5305        pxor	%xmm2, %xmm4
 5306        pxor	%xmm1, %xmm6
 5307        movdqa	%xmm4, %xmm0
 5308        movdqa	%xmm4, %xmm1
 5309        movdqa	%xmm4, %xmm2
 5310        pslld	$31, %xmm0
 5311        pslld	$30, %xmm1
 5312        pslld	$25, %xmm2
 5313        pxor	%xmm1, %xmm0
 5314        pxor	%xmm2, %xmm0
 5315        movdqa	%xmm0, %xmm1
 5316        psrldq	$4, %xmm1
 5317        pslldq	$12, %xmm0
 5318        pxor	%xmm0, %xmm4
 5319        movdqa	%xmm4, %xmm2
 5320        movdqa	%xmm4, %xmm3
 5321        movdqa	%xmm4, %xmm0
 5322        psrld	$0x01, %xmm2
 5323        psrld	$2, %xmm3
 5324        psrld	$7, %xmm0
 5325        pxor	%xmm3, %xmm2
 5326        pxor	%xmm0, %xmm2
 5327        pxor	%xmm1, %xmm2
 5328        pxor	%xmm4, %xmm2
 5329        pxor	%xmm2, %xmm6
 5330        movdqu	(%rsp), %xmm5
 5331L_AES_GCM_encrypt_update_aesni_done_128:
 5332        movl	%r8d, %edx
 5333        cmpl	%edx, %r14d
 5334        jge	L_AES_GCM_encrypt_update_aesni_done_enc
 5335        movl	%r8d, %r13d
 5336        andl	$0xfffffff0, %r13d
 5337        cmpl	%r13d, %r14d
 5338        jge	L_AES_GCM_encrypt_update_aesni_last_block_done
 5339        leaq	(%r11,%r14,1), %rcx
 5340        leaq	(%r10,%r14,1), %rdx
 5341        movdqu	(%r12), %xmm8
 5342        movdqa	%xmm8, %xmm9
 5343        pshufb	L_aes_gcm_bswap_epi64(%rip), %xmm8
 5344        paddd	L_aes_gcm_one(%rip), %xmm9
 5345        pxor	(%rdi), %xmm8
 5346        movdqu	%xmm9, (%r12)
 5347        aesenc	16(%rdi), %xmm8
 5348        aesenc	32(%rdi), %xmm8
 5349        aesenc	48(%rdi), %xmm8
 5350        aesenc	64(%rdi), %xmm8
 5351        aesenc	80(%rdi), %xmm8
 5352        aesenc	96(%rdi), %xmm8
 5353        aesenc	112(%rdi), %xmm8
 5354        aesenc	128(%rdi), %xmm8
 5355        aesenc	144(%rdi), %xmm8
 5356        cmpl	$11, %esi
 5357        movdqa	160(%rdi), %xmm9
 5358        jl	L_AES_GCM_encrypt_update_aesni_aesenc_block_aesenc_avx_last
 5359        aesenc	%xmm9, %xmm8
 5360        aesenc	176(%rdi), %xmm8
 5361        cmpl	$13, %esi
 5362        movdqa	192(%rdi), %xmm9
 5363        jl	L_AES_GCM_encrypt_update_aesni_aesenc_block_aesenc_avx_last
 5364        aesenc	%xmm9, %xmm8
 5365        aesenc	208(%rdi), %xmm8
 5366        movdqa	224(%rdi), %xmm9
 5367L_AES_GCM_encrypt_update_aesni_aesenc_block_aesenc_avx_last:
 5368        aesenclast	%xmm9, %xmm8
 5369        movdqu	(%rcx), %xmm9
 5370        pxor	%xmm9, %xmm8
 5371        movdqu	%xmm8, (%rdx)
 5372        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm8
 5373        pxor	%xmm8, %xmm6
 5374        addl	$16, %r14d
 5375        cmpl	%r13d, %r14d
 5376        jge	L_AES_GCM_encrypt_update_aesni_last_block_ghash
 5377L_AES_GCM_encrypt_update_aesni_last_block_start:
 5378        leaq	(%r11,%r14,1), %rcx
 5379        leaq	(%r10,%r14,1), %rdx
 5380        movdqu	(%r12), %xmm8
 5381        movdqa	%xmm8, %xmm9
 5382        pshufb	L_aes_gcm_bswap_epi64(%rip), %xmm8
 5383        paddd	L_aes_gcm_one(%rip), %xmm9
 5384        pxor	(%rdi), %xmm8
 5385        movdqu	%xmm9, (%r12)
 5386        movdqa	%xmm6, %xmm10
 5387        pclmulqdq	$16, %xmm5, %xmm10
 5388        aesenc	16(%rdi), %xmm8
 5389        aesenc	32(%rdi), %xmm8
 5390        movdqa	%xmm6, %xmm11
 5391        pclmulqdq	$0x01, %xmm5, %xmm11
 5392        aesenc	48(%rdi), %xmm8
 5393        aesenc	64(%rdi), %xmm8
 5394        movdqa	%xmm6, %xmm12
 5395        pclmulqdq	$0x00, %xmm5, %xmm12
 5396        aesenc	80(%rdi), %xmm8
 5397        movdqa	%xmm6, %xmm1
 5398        pclmulqdq	$0x11, %xmm5, %xmm1
 5399        aesenc	96(%rdi), %xmm8
 5400        pxor	%xmm11, %xmm10
 5401        movdqa	%xmm10, %xmm2
 5402        psrldq	$8, %xmm10
 5403        pslldq	$8, %xmm2
 5404        aesenc	112(%rdi), %xmm8
 5405        movdqa	%xmm1, %xmm3
 5406        pxor	%xmm12, %xmm2
 5407        pxor	%xmm10, %xmm3
 5408        movdqa	L_aes_gcm_mod2_128(%rip), %xmm0
 5409        movdqa	%xmm2, %xmm11
 5410        pclmulqdq	$16, %xmm0, %xmm11
 5411        aesenc	128(%rdi), %xmm8
 5412        pshufd	$0x4e, %xmm2, %xmm10
 5413        pxor	%xmm11, %xmm10
 5414        movdqa	%xmm10, %xmm11
 5415        pclmulqdq	$16, %xmm0, %xmm11
 5416        aesenc	144(%rdi), %xmm8
 5417        pshufd	$0x4e, %xmm10, %xmm6
 5418        pxor	%xmm11, %xmm6
 5419        pxor	%xmm3, %xmm6
 5420        cmpl	$11, %esi
 5421        movdqa	160(%rdi), %xmm9
 5422        jl	L_AES_GCM_encrypt_update_aesni_aesenc_gfmul_last
 5423        aesenc	%xmm9, %xmm8
 5424        aesenc	176(%rdi), %xmm8
 5425        cmpl	$13, %esi
 5426        movdqa	192(%rdi), %xmm9
 5427        jl	L_AES_GCM_encrypt_update_aesni_aesenc_gfmul_last
 5428        aesenc	%xmm9, %xmm8
 5429        aesenc	208(%rdi), %xmm8
 5430        movdqa	224(%rdi), %xmm9
 5431L_AES_GCM_encrypt_update_aesni_aesenc_gfmul_last:
 5432        aesenclast	%xmm9, %xmm8
 5433        movdqu	(%rcx), %xmm9
 5434        pxor	%xmm9, %xmm8
 5435        movdqu	%xmm8, (%rdx)
 5436        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm8
 5437        pxor	%xmm8, %xmm6
 5438        addl	$16, %r14d
 5439        cmpl	%r13d, %r14d
 5440        jl	L_AES_GCM_encrypt_update_aesni_last_block_start
 5441L_AES_GCM_encrypt_update_aesni_last_block_ghash:
 5442        pshufd	$0x4e, %xmm5, %xmm9
 5443        pshufd	$0x4e, %xmm6, %xmm10
 5444        movdqa	%xmm6, %xmm11
 5445        movdqa	%xmm6, %xmm8
 5446        pclmulqdq	$0x11, %xmm5, %xmm11
 5447        pclmulqdq	$0x00, %xmm5, %xmm8
 5448        pxor	%xmm5, %xmm9
 5449        pxor	%xmm6, %xmm10
 5450        pclmulqdq	$0x00, %xmm10, %xmm9
 5451        pxor	%xmm8, %xmm9
 5452        pxor	%xmm11, %xmm9
 5453        movdqa	%xmm9, %xmm10
 5454        movdqa	%xmm11, %xmm6
 5455        pslldq	$8, %xmm10
 5456        psrldq	$8, %xmm9
 5457        pxor	%xmm10, %xmm8
 5458        pxor	%xmm9, %xmm6
 5459        movdqa	%xmm8, %xmm12
 5460        movdqa	%xmm8, %xmm13
 5461        movdqa	%xmm8, %xmm14
 5462        pslld	$31, %xmm12
 5463        pslld	$30, %xmm13
 5464        pslld	$25, %xmm14
 5465        pxor	%xmm13, %xmm12
 5466        pxor	%xmm14, %xmm12
 5467        movdqa	%xmm12, %xmm13
 5468        psrldq	$4, %xmm13
 5469        pslldq	$12, %xmm12
 5470        pxor	%xmm12, %xmm8
 5471        movdqa	%xmm8, %xmm14
 5472        movdqa	%xmm8, %xmm10
 5473        movdqa	%xmm8, %xmm9
 5474        psrld	$0x01, %xmm14
 5475        psrld	$2, %xmm10
 5476        psrld	$7, %xmm9
 5477        pxor	%xmm10, %xmm14
 5478        pxor	%xmm9, %xmm14
 5479        pxor	%xmm13, %xmm14
 5480        pxor	%xmm8, %xmm14
 5481        pxor	%xmm14, %xmm6
 5482L_AES_GCM_encrypt_update_aesni_last_block_done:
 5483L_AES_GCM_encrypt_update_aesni_done_enc:
 5484        movdqa	%xmm6, (%r9)
 5485        addq	$0xa0, %rsp
 5486        popq	%r14
 5487        popq	%r12
 5488        popq	%r13
 5489        repz retq
 5490#ifndef __APPLE__
 5491.size	AES_GCM_encrypt_update_aesni,.-AES_GCM_encrypt_update_aesni
 5492#endif /* __APPLE__ */
 5493#ifndef __APPLE__
 5494.text
 5495.globl	AES_GCM_encrypt_final_aesni
 5496.type	AES_GCM_encrypt_final_aesni,@function
 5497.align	16
 5498AES_GCM_encrypt_final_aesni:
 5499#else
 5500.section	__TEXT,__text
 5501.globl	_AES_GCM_encrypt_final_aesni
 5502.p2align	4
 5503_AES_GCM_encrypt_final_aesni:
 5504#endif /* __APPLE__ */
 5505        pushq	%r13
 5506        movl	%edx, %eax
 5507        movl	%ecx, %r10d
 5508        movl	%r8d, %r11d
 5509        movq	16(%rsp), %r8
 5510        subq	$16, %rsp
 5511        movdqa	(%rdi), %xmm4
 5512        movdqa	(%r9), %xmm5
 5513        movdqa	(%r8), %xmm6
 5514        movdqa	%xmm5, %xmm8
 5515        movdqa	%xmm5, %xmm7
 5516        psrlq	$63, %xmm8
 5517        psllq	$0x01, %xmm7
 5518        pslldq	$8, %xmm8
 5519        por	%xmm8, %xmm7
 5520        pshufd	$0xff, %xmm5, %xmm5
 5521        psrad	$31, %xmm5
 5522        pand	L_aes_gcm_mod2_128(%rip), %xmm5
 5523        pxor	%xmm7, %xmm5
 5524        movl	%r10d, %edx
 5525        movl	%r11d, %ecx
 5526        shlq	$3, %rdx
 5527        shlq	$3, %rcx
 5528        pinsrq	$0x00, %rdx, %xmm0
 5529        pinsrq	$0x01, %rcx, %xmm0
 5530        pxor	%xmm0, %xmm4
 5531        pshufd	$0x4e, %xmm5, %xmm8
 5532        pshufd	$0x4e, %xmm4, %xmm9
 5533        movdqa	%xmm4, %xmm10
 5534        movdqa	%xmm4, %xmm7
 5535        pclmulqdq	$0x11, %xmm5, %xmm10
 5536        pclmulqdq	$0x00, %xmm5, %xmm7
 5537        pxor	%xmm5, %xmm8
 5538        pxor	%xmm4, %xmm9
 5539        pclmulqdq	$0x00, %xmm9, %xmm8
 5540        pxor	%xmm7, %xmm8
 5541        pxor	%xmm10, %xmm8
 5542        movdqa	%xmm8, %xmm9
 5543        movdqa	%xmm10, %xmm4
 5544        pslldq	$8, %xmm9
 5545        psrldq	$8, %xmm8
 5546        pxor	%xmm9, %xmm7
 5547        pxor	%xmm8, %xmm4
 5548        movdqa	%xmm7, %xmm11
 5549        movdqa	%xmm7, %xmm12
 5550        movdqa	%xmm7, %xmm13
 5551        pslld	$31, %xmm11
 5552        pslld	$30, %xmm12
 5553        pslld	$25, %xmm13
 5554        pxor	%xmm12, %xmm11
 5555        pxor	%xmm13, %xmm11
 5556        movdqa	%xmm11, %xmm12
 5557        psrldq	$4, %xmm12
 5558        pslldq	$12, %xmm11
 5559        pxor	%xmm11, %xmm7
 5560        movdqa	%xmm7, %xmm13
 5561        movdqa	%xmm7, %xmm9
 5562        movdqa	%xmm7, %xmm8
 5563        psrld	$0x01, %xmm13
 5564        psrld	$2, %xmm9
 5565        psrld	$7, %xmm8
 5566        pxor	%xmm9, %xmm13
 5567        pxor	%xmm8, %xmm13
 5568        pxor	%xmm12, %xmm13
 5569        pxor	%xmm7, %xmm13
 5570        pxor	%xmm13, %xmm4
 5571        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm4
 5572        movdqu	%xmm6, %xmm0
 5573        pxor	%xmm4, %xmm0
 5574        cmpl	$16, %eax
 5575        je	L_AES_GCM_encrypt_final_aesni_store_tag_16
 5576        xorq	%rcx, %rcx
 5577        movdqu	%xmm0, (%rsp)
 5578L_AES_GCM_encrypt_final_aesni_store_tag_loop:
 5579        movzbl	(%rsp,%rcx,1), %r13d
 5580        movb	%r13b, (%rsi,%rcx,1)
 5581        incl	%ecx
 5582        cmpl	%eax, %ecx
 5583        jne	L_AES_GCM_encrypt_final_aesni_store_tag_loop
 5584        jmp	L_AES_GCM_encrypt_final_aesni_store_tag_done
 5585L_AES_GCM_encrypt_final_aesni_store_tag_16:
 5586        movdqu	%xmm0, (%rsi)
 5587L_AES_GCM_encrypt_final_aesni_store_tag_done:
 5588        addq	$16, %rsp
 5589        popq	%r13
 5590        repz retq
 5591#ifndef __APPLE__
 5592.size	AES_GCM_encrypt_final_aesni,.-AES_GCM_encrypt_final_aesni
 5593#endif /* __APPLE__ */
 5594#ifndef __APPLE__
 5595.text
 5596.globl	AES_GCM_decrypt_update_aesni
 5597.type	AES_GCM_decrypt_update_aesni,@function
 5598.align	16
 5599AES_GCM_decrypt_update_aesni:
 5600#else
 5601.section	__TEXT,__text
 5602.globl	_AES_GCM_decrypt_update_aesni
 5603.p2align	4
 5604_AES_GCM_decrypt_update_aesni:
 5605#endif /* __APPLE__ */
 5606        pushq	%r13
 5607        pushq	%r12
 5608        pushq	%r14
 5609        pushq	%r15
 5610        movq	%rdx, %r10
 5611        movq	%rcx, %r11
 5612        movq	40(%rsp), %rax
 5613        movq	48(%rsp), %r12
 5614        subq	$0xa8, %rsp
 5615        movdqa	(%r9), %xmm6
 5616        movdqa	(%rax), %xmm5
 5617        movdqa	%xmm5, %xmm9
 5618        movdqa	%xmm5, %xmm8
 5619        psrlq	$63, %xmm9
 5620        psllq	$0x01, %xmm8
 5621        pslldq	$8, %xmm9
 5622        por	%xmm9, %xmm8
 5623        pshufd	$0xff, %xmm5, %xmm5
 5624        psrad	$31, %xmm5
 5625        pand	L_aes_gcm_mod2_128(%rip), %xmm5
 5626        pxor	%xmm8, %xmm5
 5627        xorl	%r14d, %r14d
 5628        cmpl	$0x80, %r8d
 5629        movl	%r8d, %r13d
 5630        jl	L_AES_GCM_decrypt_update_aesni_done_128
 5631        andl	$0xffffff80, %r13d
 5632        movdqa	%xmm6, %xmm2
 5633        # H ^ 1
 5634        movdqu	%xmm5, (%rsp)
 5635        # H ^ 2
 5636        pshufd	$0x4e, %xmm5, %xmm9
 5637        pshufd	$0x4e, %xmm5, %xmm10
 5638        movdqa	%xmm5, %xmm11
 5639        movdqa	%xmm5, %xmm8
 5640        pclmulqdq	$0x11, %xmm5, %xmm11
 5641        pclmulqdq	$0x00, %xmm5, %xmm8
 5642        pxor	%xmm5, %xmm9
 5643        pxor	%xmm5, %xmm10
 5644        pclmulqdq	$0x00, %xmm10, %xmm9
 5645        pxor	%xmm8, %xmm9
 5646        pxor	%xmm11, %xmm9
 5647        movdqa	%xmm9, %xmm10
 5648        movdqa	%xmm11, %xmm0
 5649        pslldq	$8, %xmm10
 5650        psrldq	$8, %xmm9
 5651        pxor	%xmm10, %xmm8
 5652        pxor	%xmm9, %xmm0
 5653        movdqa	%xmm8, %xmm12
 5654        movdqa	%xmm8, %xmm13
 5655        movdqa	%xmm8, %xmm14
 5656        pslld	$31, %xmm12
 5657        pslld	$30, %xmm13
 5658        pslld	$25, %xmm14
 5659        pxor	%xmm13, %xmm12
 5660        pxor	%xmm14, %xmm12
 5661        movdqa	%xmm12, %xmm13
 5662        psrldq	$4, %xmm13
 5663        pslldq	$12, %xmm12
 5664        pxor	%xmm12, %xmm8
 5665        movdqa	%xmm8, %xmm14
 5666        movdqa	%xmm8, %xmm10
 5667        movdqa	%xmm8, %xmm9
 5668        psrld	$0x01, %xmm14
 5669        psrld	$2, %xmm10
 5670        psrld	$7, %xmm9
 5671        pxor	%xmm10, %xmm14
 5672        pxor	%xmm9, %xmm14
 5673        pxor	%xmm13, %xmm14
 5674        pxor	%xmm8, %xmm14
 5675        pxor	%xmm14, %xmm0
 5676        movdqu	%xmm0, 16(%rsp)
 5677        # H ^ 3
 5678        pshufd	$0x4e, %xmm5, %xmm9
 5679        pshufd	$0x4e, %xmm0, %xmm10
 5680        movdqa	%xmm0, %xmm11
 5681        movdqa	%xmm0, %xmm8
 5682        pclmulqdq	$0x11, %xmm5, %xmm11
 5683        pclmulqdq	$0x00, %xmm5, %xmm8
 5684        pxor	%xmm5, %xmm9
 5685        pxor	%xmm0, %xmm10
 5686        pclmulqdq	$0x00, %xmm10, %xmm9
 5687        pxor	%xmm8, %xmm9
 5688        pxor	%xmm11, %xmm9
 5689        movdqa	%xmm9, %xmm10
 5690        movdqa	%xmm11, %xmm1
 5691        pslldq	$8, %xmm10
 5692        psrldq	$8, %xmm9
 5693        pxor	%xmm10, %xmm8
 5694        pxor	%xmm9, %xmm1
 5695        movdqa	%xmm8, %xmm12
 5696        movdqa	%xmm8, %xmm13
 5697        movdqa	%xmm8, %xmm14
 5698        pslld	$31, %xmm12
 5699        pslld	$30, %xmm13
 5700        pslld	$25, %xmm14
 5701        pxor	%xmm13, %xmm12
 5702        pxor	%xmm14, %xmm12
 5703        movdqa	%xmm12, %xmm13
 5704        psrldq	$4, %xmm13
 5705        pslldq	$12, %xmm12
 5706        pxor	%xmm12, %xmm8
 5707        movdqa	%xmm8, %xmm14
 5708        movdqa	%xmm8, %xmm10
 5709        movdqa	%xmm8, %xmm9
 5710        psrld	$0x01, %xmm14
 5711        psrld	$2, %xmm10
 5712        psrld	$7, %xmm9
 5713        pxor	%xmm10, %xmm14
 5714        pxor	%xmm9, %xmm14
 5715        pxor	%xmm13, %xmm14
 5716        pxor	%xmm8, %xmm14
 5717        pxor	%xmm14, %xmm1
 5718        movdqu	%xmm1, 32(%rsp)
 5719        # H ^ 4
 5720        pshufd	$0x4e, %xmm0, %xmm9
 5721        pshufd	$0x4e, %xmm0, %xmm10
 5722        movdqa	%xmm0, %xmm11
 5723        movdqa	%xmm0, %xmm8
 5724        pclmulqdq	$0x11, %xmm0, %xmm11
 5725        pclmulqdq	$0x00, %xmm0, %xmm8
 5726        pxor	%xmm0, %xmm9
 5727        pxor	%xmm0, %xmm10
 5728        pclmulqdq	$0x00, %xmm10, %xmm9
 5729        pxor	%xmm8, %xmm9
 5730        pxor	%xmm11, %xmm9
 5731        movdqa	%xmm9, %xmm10
 5732        movdqa	%xmm11, %xmm3
 5733        pslldq	$8, %xmm10
 5734        psrldq	$8, %xmm9
 5735        pxor	%xmm10, %xmm8
 5736        pxor	%xmm9, %xmm3
 5737        movdqa	%xmm8, %xmm12
 5738        movdqa	%xmm8, %xmm13
 5739        movdqa	%xmm8, %xmm14
 5740        pslld	$31, %xmm12
 5741        pslld	$30, %xmm13
 5742        pslld	$25, %xmm14
 5743        pxor	%xmm13, %xmm12
 5744        pxor	%xmm14, %xmm12
 5745        movdqa	%xmm12, %xmm13
 5746        psrldq	$4, %xmm13
 5747        pslldq	$12, %xmm12
 5748        pxor	%xmm12, %xmm8
 5749        movdqa	%xmm8, %xmm14
 5750        movdqa	%xmm8, %xmm10
 5751        movdqa	%xmm8, %xmm9
 5752        psrld	$0x01, %xmm14
 5753        psrld	$2, %xmm10
 5754        psrld	$7, %xmm9
 5755        pxor	%xmm10, %xmm14
 5756        pxor	%xmm9, %xmm14
 5757        pxor	%xmm13, %xmm14
 5758        pxor	%xmm8, %xmm14
 5759        pxor	%xmm14, %xmm3
 5760        movdqu	%xmm3, 48(%rsp)
 5761        # H ^ 5
 5762        pshufd	$0x4e, %xmm0, %xmm9
 5763        pshufd	$0x4e, %xmm1, %xmm10
 5764        movdqa	%xmm1, %xmm11
 5765        movdqa	%xmm1, %xmm8
 5766        pclmulqdq	$0x11, %xmm0, %xmm11
 5767        pclmulqdq	$0x00, %xmm0, %xmm8
 5768        pxor	%xmm0, %xmm9
 5769        pxor	%xmm1, %xmm10
 5770        pclmulqdq	$0x00, %xmm10, %xmm9
 5771        pxor	%xmm8, %xmm9
 5772        pxor	%xmm11, %xmm9
 5773        movdqa	%xmm9, %xmm10
 5774        movdqa	%xmm11, %xmm7
 5775        pslldq	$8, %xmm10
 5776        psrldq	$8, %xmm9
 5777        pxor	%xmm10, %xmm8
 5778        pxor	%xmm9, %xmm7
 5779        movdqa	%xmm8, %xmm12
 5780        movdqa	%xmm8, %xmm13
 5781        movdqa	%xmm8, %xmm14
 5782        pslld	$31, %xmm12
 5783        pslld	$30, %xmm13
 5784        pslld	$25, %xmm14
 5785        pxor	%xmm13, %xmm12
 5786        pxor	%xmm14, %xmm12
 5787        movdqa	%xmm12, %xmm13
 5788        psrldq	$4, %xmm13
 5789        pslldq	$12, %xmm12
 5790        pxor	%xmm12, %xmm8
 5791        movdqa	%xmm8, %xmm14
 5792        movdqa	%xmm8, %xmm10
 5793        movdqa	%xmm8, %xmm9
 5794        psrld	$0x01, %xmm14
 5795        psrld	$2, %xmm10
 5796        psrld	$7, %xmm9
 5797        pxor	%xmm10, %xmm14
 5798        pxor	%xmm9, %xmm14
 5799        pxor	%xmm13, %xmm14
 5800        pxor	%xmm8, %xmm14
 5801        pxor	%xmm14, %xmm7
 5802        movdqu	%xmm7, 64(%rsp)
 5803        # H ^ 6
 5804        pshufd	$0x4e, %xmm1, %xmm9
 5805        pshufd	$0x4e, %xmm1, %xmm10
 5806        movdqa	%xmm1, %xmm11
 5807        movdqa	%xmm1, %xmm8
 5808        pclmulqdq	$0x11, %xmm1, %xmm11
 5809        pclmulqdq	$0x00, %xmm1, %xmm8
 5810        pxor	%xmm1, %xmm9
 5811        pxor	%xmm1, %xmm10
 5812        pclmulqdq	$0x00, %xmm10, %xmm9
 5813        pxor	%xmm8, %xmm9
 5814        pxor	%xmm11, %xmm9
 5815        movdqa	%xmm9, %xmm10
 5816        movdqa	%xmm11, %xmm7
 5817        pslldq	$8, %xmm10
 5818        psrldq	$8, %xmm9
 5819        pxor	%xmm10, %xmm8
 5820        pxor	%xmm9, %xmm7
 5821        movdqa	%xmm8, %xmm12
 5822        movdqa	%xmm8, %xmm13
 5823        movdqa	%xmm8, %xmm14
 5824        pslld	$31, %xmm12
 5825        pslld	$30, %xmm13
 5826        pslld	$25, %xmm14
 5827        pxor	%xmm13, %xmm12
 5828        pxor	%xmm14, %xmm12
 5829        movdqa	%xmm12, %xmm13
 5830        psrldq	$4, %xmm13
 5831        pslldq	$12, %xmm12
 5832        pxor	%xmm12, %xmm8
 5833        movdqa	%xmm8, %xmm14
 5834        movdqa	%xmm8, %xmm10
 5835        movdqa	%xmm8, %xmm9
 5836        psrld	$0x01, %xmm14
 5837        psrld	$2, %xmm10
 5838        psrld	$7, %xmm9
 5839        pxor	%xmm10, %xmm14
 5840        pxor	%xmm9, %xmm14
 5841        pxor	%xmm13, %xmm14
 5842        pxor	%xmm8, %xmm14
 5843        pxor	%xmm14, %xmm7
 5844        movdqu	%xmm7, 80(%rsp)
 5845        # H ^ 7
 5846        pshufd	$0x4e, %xmm1, %xmm9
 5847        pshufd	$0x4e, %xmm3, %xmm10
 5848        movdqa	%xmm3, %xmm11
 5849        movdqa	%xmm3, %xmm8
 5850        pclmulqdq	$0x11, %xmm1, %xmm11
 5851        pclmulqdq	$0x00, %xmm1, %xmm8
 5852        pxor	%xmm1, %xmm9
 5853        pxor	%xmm3, %xmm10
 5854        pclmulqdq	$0x00, %xmm10, %xmm9
 5855        pxor	%xmm8, %xmm9
 5856        pxor	%xmm11, %xmm9
 5857        movdqa	%xmm9, %xmm10
 5858        movdqa	%xmm11, %xmm7
 5859        pslldq	$8, %xmm10
 5860        psrldq	$8, %xmm9
 5861        pxor	%xmm10, %xmm8
 5862        pxor	%xmm9, %xmm7
 5863        movdqa	%xmm8, %xmm12
 5864        movdqa	%xmm8, %xmm13
 5865        movdqa	%xmm8, %xmm14
 5866        pslld	$31, %xmm12
 5867        pslld	$30, %xmm13
 5868        pslld	$25, %xmm14
 5869        pxor	%xmm13, %xmm12
 5870        pxor	%xmm14, %xmm12
 5871        movdqa	%xmm12, %xmm13
 5872        psrldq	$4, %xmm13
 5873        pslldq	$12, %xmm12
 5874        pxor	%xmm12, %xmm8
 5875        movdqa	%xmm8, %xmm14
 5876        movdqa	%xmm8, %xmm10
 5877        movdqa	%xmm8, %xmm9
 5878        psrld	$0x01, %xmm14
 5879        psrld	$2, %xmm10
 5880        psrld	$7, %xmm9
 5881        pxor	%xmm10, %xmm14
 5882        pxor	%xmm9, %xmm14
 5883        pxor	%xmm13, %xmm14
 5884        pxor	%xmm8, %xmm14
 5885        pxor	%xmm14, %xmm7
 5886        movdqu	%xmm7, 96(%rsp)
 5887        # H ^ 8
 5888        pshufd	$0x4e, %xmm3, %xmm9
 5889        pshufd	$0x4e, %xmm3, %xmm10
 5890        movdqa	%xmm3, %xmm11
 5891        movdqa	%xmm3, %xmm8
 5892        pclmulqdq	$0x11, %xmm3, %xmm11
 5893        pclmulqdq	$0x00, %xmm3, %xmm8
 5894        pxor	%xmm3, %xmm9
 5895        pxor	%xmm3, %xmm10
 5896        pclmulqdq	$0x00, %xmm10, %xmm9
 5897        pxor	%xmm8, %xmm9
 5898        pxor	%xmm11, %xmm9
 5899        movdqa	%xmm9, %xmm10
 5900        movdqa	%xmm11, %xmm7
 5901        pslldq	$8, %xmm10
 5902        psrldq	$8, %xmm9
 5903        pxor	%xmm10, %xmm8
 5904        pxor	%xmm9, %xmm7
 5905        movdqa	%xmm8, %xmm12
 5906        movdqa	%xmm8, %xmm13
 5907        movdqa	%xmm8, %xmm14
 5908        pslld	$31, %xmm12
 5909        pslld	$30, %xmm13
 5910        pslld	$25, %xmm14
 5911        pxor	%xmm13, %xmm12
 5912        pxor	%xmm14, %xmm12
 5913        movdqa	%xmm12, %xmm13
 5914        psrldq	$4, %xmm13
 5915        pslldq	$12, %xmm12
 5916        pxor	%xmm12, %xmm8
 5917        movdqa	%xmm8, %xmm14
 5918        movdqa	%xmm8, %xmm10
 5919        movdqa	%xmm8, %xmm9
 5920        psrld	$0x01, %xmm14
 5921        psrld	$2, %xmm10
 5922        psrld	$7, %xmm9
 5923        pxor	%xmm10, %xmm14
 5924        pxor	%xmm9, %xmm14
 5925        pxor	%xmm13, %xmm14
 5926        pxor	%xmm8, %xmm14
 5927        pxor	%xmm14, %xmm7
 5928        movdqu	%xmm7, 112(%rsp)
 5929L_AES_GCM_decrypt_update_aesni_ghash_128:
 5930        leaq	(%r11,%r14,1), %rcx
 5931        leaq	(%r10,%r14,1), %rdx
 5932        movdqu	(%r12), %xmm8
 5933        movdqa	L_aes_gcm_bswap_epi64(%rip), %xmm1
 5934        movdqa	%xmm8, %xmm0
 5935        pshufb	%xmm1, %xmm8
 5936        movdqa	%xmm0, %xmm9
 5937        paddd	L_aes_gcm_one(%rip), %xmm9
 5938        pshufb	%xmm1, %xmm9
 5939        movdqa	%xmm0, %xmm10
 5940        paddd	L_aes_gcm_two(%rip), %xmm10
 5941        pshufb	%xmm1, %xmm10
 5942        movdqa	%xmm0, %xmm11
 5943        paddd	L_aes_gcm_three(%rip), %xmm11
 5944        pshufb	%xmm1, %xmm11
 5945        movdqa	%xmm0, %xmm12
 5946        paddd	L_aes_gcm_four(%rip), %xmm12
 5947        pshufb	%xmm1, %xmm12
 5948        movdqa	%xmm0, %xmm13
 5949        paddd	L_aes_gcm_five(%rip), %xmm13
 5950        pshufb	%xmm1, %xmm13
 5951        movdqa	%xmm0, %xmm14
 5952        paddd	L_aes_gcm_six(%rip), %xmm14
 5953        pshufb	%xmm1, %xmm14
 5954        movdqa	%xmm0, %xmm15
 5955        paddd	L_aes_gcm_seven(%rip), %xmm15
 5956        pshufb	%xmm1, %xmm15
 5957        paddd	L_aes_gcm_eight(%rip), %xmm0
 5958        movdqa	(%rdi), %xmm7
 5959        movdqu	%xmm0, (%r12)
 5960        pxor	%xmm7, %xmm8
 5961        pxor	%xmm7, %xmm9
 5962        pxor	%xmm7, %xmm10
 5963        pxor	%xmm7, %xmm11
 5964        pxor	%xmm7, %xmm12
 5965        pxor	%xmm7, %xmm13
 5966        pxor	%xmm7, %xmm14
 5967        pxor	%xmm7, %xmm15
 5968        movdqu	112(%rsp), %xmm7
 5969        movdqu	(%rcx), %xmm0
 5970        aesenc	16(%rdi), %xmm8
 5971        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm0
 5972        pxor	%xmm2, %xmm0
 5973        pshufd	$0x4e, %xmm7, %xmm1
 5974        pshufd	$0x4e, %xmm0, %xmm5
 5975        pxor	%xmm7, %xmm1
 5976        pxor	%xmm0, %xmm5
 5977        movdqa	%xmm0, %xmm3
 5978        pclmulqdq	$0x11, %xmm7, %xmm3
 5979        aesenc	16(%rdi), %xmm9
 5980        aesenc	16(%rdi), %xmm10
 5981        movdqa	%xmm0, %xmm2
 5982        pclmulqdq	$0x00, %xmm7, %xmm2
 5983        aesenc	16(%rdi), %xmm11
 5984        aesenc	16(%rdi), %xmm12
 5985        pclmulqdq	$0x00, %xmm5, %xmm1
 5986        aesenc	16(%rdi), %xmm13
 5987        aesenc	16(%rdi), %xmm14
 5988        aesenc	16(%rdi), %xmm15
 5989        pxor	%xmm2, %xmm1
 5990        pxor	%xmm3, %xmm1
 5991        movdqu	96(%rsp), %xmm7
 5992        movdqu	16(%rcx), %xmm0
 5993        pshufd	$0x4e, %xmm7, %xmm4
 5994        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm0
 5995        aesenc	32(%rdi), %xmm8
 5996        pxor	%xmm7, %xmm4
 5997        pshufd	$0x4e, %xmm0, %xmm5
 5998        pxor	%xmm0, %xmm5
 5999        movdqa	%xmm0, %xmm6
 6000        pclmulqdq	$0x11, %xmm7, %xmm6
 6001        aesenc	32(%rdi), %xmm9
 6002        aesenc	32(%rdi), %xmm10
 6003        pclmulqdq	$0x00, %xmm0, %xmm7
 6004        aesenc	32(%rdi), %xmm11
 6005        aesenc	32(%rdi), %xmm12
 6006        pclmulqdq	$0x00, %xmm5, %xmm4
 6007        aesenc	32(%rdi), %xmm13
 6008        aesenc	32(%rdi), %xmm14
 6009        aesenc	32(%rdi), %xmm15
 6010        pxor	%xmm7, %xmm1
 6011        pxor	%xmm7, %xmm2
 6012        pxor	%xmm6, %xmm1
 6013        pxor	%xmm6, %xmm3
 6014        pxor	%xmm4, %xmm1
 6015        movdqu	80(%rsp), %xmm7
 6016        movdqu	32(%rcx), %xmm0
 6017        pshufd	$0x4e, %xmm7, %xmm4
 6018        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm0
 6019        aesenc	48(%rdi), %xmm8
 6020        pxor	%xmm7, %xmm4
 6021        pshufd	$0x4e, %xmm0, %xmm5
 6022        pxor	%xmm0, %xmm5
 6023        movdqa	%xmm0, %xmm6
 6024        pclmulqdq	$0x11, %xmm7, %xmm6
 6025        aesenc	48(%rdi), %xmm9
 6026        aesenc	48(%rdi), %xmm10
 6027        pclmulqdq	$0x00, %xmm0, %xmm7
 6028        aesenc	48(%rdi), %xmm11
 6029        aesenc	48(%rdi), %xmm12
 6030        pclmulqdq	$0x00, %xmm5, %xmm4
 6031        aesenc	48(%rdi), %xmm13
 6032        aesenc	48(%rdi), %xmm14
 6033        aesenc	48(%rdi), %xmm15
 6034        pxor	%xmm7, %xmm1
 6035        pxor	%xmm7, %xmm2
 6036        pxor	%xmm6, %xmm1
 6037        pxor	%xmm6, %xmm3
 6038        pxor	%xmm4, %xmm1
 6039        movdqu	64(%rsp), %xmm7
 6040        movdqu	48(%rcx), %xmm0
 6041        pshufd	$0x4e, %xmm7, %xmm4
 6042        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm0
 6043        aesenc	64(%rdi), %xmm8
 6044        pxor	%xmm7, %xmm4
 6045        pshufd	$0x4e, %xmm0, %xmm5
 6046        pxor	%xmm0, %xmm5
 6047        movdqa	%xmm0, %xmm6
 6048        pclmulqdq	$0x11, %xmm7, %xmm6
 6049        aesenc	64(%rdi), %xmm9
 6050        aesenc	64(%rdi), %xmm10
 6051        pclmulqdq	$0x00, %xmm0, %xmm7
 6052        aesenc	64(%rdi), %xmm11
 6053        aesenc	64(%rdi), %xmm12
 6054        pclmulqdq	$0x00, %xmm5, %xmm4
 6055        aesenc	64(%rdi), %xmm13
 6056        aesenc	64(%rdi), %xmm14
 6057        aesenc	64(%rdi), %xmm15
 6058        pxor	%xmm7, %xmm1
 6059        pxor	%xmm7, %xmm2
 6060        pxor	%xmm6, %xmm1
 6061        pxor	%xmm6, %xmm3
 6062        pxor	%xmm4, %xmm1
 6063        movdqu	48(%rsp), %xmm7
 6064        movdqu	64(%rcx), %xmm0
 6065        pshufd	$0x4e, %xmm7, %xmm4
 6066        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm0
 6067        aesenc	80(%rdi), %xmm8
 6068        pxor	%xmm7, %xmm4
 6069        pshufd	$0x4e, %xmm0, %xmm5
 6070        pxor	%xmm0, %xmm5
 6071        movdqa	%xmm0, %xmm6
 6072        pclmulqdq	$0x11, %xmm7, %xmm6
 6073        aesenc	80(%rdi), %xmm9
 6074        aesenc	80(%rdi), %xmm10
 6075        pclmulqdq	$0x00, %xmm0, %xmm7
 6076        aesenc	80(%rdi), %xmm11
 6077        aesenc	80(%rdi), %xmm12
 6078        pclmulqdq	$0x00, %xmm5, %xmm4
 6079        aesenc	80(%rdi), %xmm13
 6080        aesenc	80(%rdi), %xmm14
 6081        aesenc	80(%rdi), %xmm15
 6082        pxor	%xmm7, %xmm1
 6083        pxor	%xmm7, %xmm2
 6084        pxor	%xmm6, %xmm1
 6085        pxor	%xmm6, %xmm3
 6086        pxor	%xmm4, %xmm1
 6087        movdqu	32(%rsp), %xmm7
 6088        movdqu	80(%rcx), %xmm0
 6089        pshufd	$0x4e, %xmm7, %xmm4
 6090        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm0
 6091        aesenc	96(%rdi), %xmm8
 6092        pxor	%xmm7, %xmm4
 6093        pshufd	$0x4e, %xmm0, %xmm5
 6094        pxor	%xmm0, %xmm5
 6095        movdqa	%xmm0, %xmm6
 6096        pclmulqdq	$0x11, %xmm7, %xmm6
 6097        aesenc	96(%rdi), %xmm9
 6098        aesenc	96(%rdi), %xmm10
 6099        pclmulqdq	$0x00, %xmm0, %xmm7
 6100        aesenc	96(%rdi), %xmm11
 6101        aesenc	96(%rdi), %xmm12
 6102        pclmulqdq	$0x00, %xmm5, %xmm4
 6103        aesenc	96(%rdi), %xmm13
 6104        aesenc	96(%rdi), %xmm14
 6105        aesenc	96(%rdi), %xmm15
 6106        pxor	%xmm7, %xmm1
 6107        pxor	%xmm7, %xmm2
 6108        pxor	%xmm6, %xmm1
 6109        pxor	%xmm6, %xmm3
 6110        pxor	%xmm4, %xmm1
 6111        movdqu	16(%rsp), %xmm7
 6112        movdqu	96(%rcx), %xmm0
 6113        pshufd	$0x4e, %xmm7, %xmm4
 6114        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm0
 6115        aesenc	112(%rdi), %xmm8
 6116        pxor	%xmm7, %xmm4
 6117        pshufd	$0x4e, %xmm0, %xmm5
 6118        pxor	%xmm0, %xmm5
 6119        movdqa	%xmm0, %xmm6
 6120        pclmulqdq	$0x11, %xmm7, %xmm6
 6121        aesenc	112(%rdi), %xmm9
 6122        aesenc	112(%rdi), %xmm10
 6123        pclmulqdq	$0x00, %xmm0, %xmm7
 6124        aesenc	112(%rdi), %xmm11
 6125        aesenc	112(%rdi), %xmm12
 6126        pclmulqdq	$0x00, %xmm5, %xmm4
 6127        aesenc	112(%rdi), %xmm13
 6128        aesenc	112(%rdi), %xmm14
 6129        aesenc	112(%rdi), %xmm15
 6130        pxor	%xmm7, %xmm1
 6131        pxor	%xmm7, %xmm2
 6132        pxor	%xmm6, %xmm1
 6133        pxor	%xmm6, %xmm3
 6134        pxor	%xmm4, %xmm1
 6135        movdqu	(%rsp), %xmm7
 6136        movdqu	112(%rcx), %xmm0
 6137        pshufd	$0x4e, %xmm7, %xmm4
 6138        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm0
 6139        aesenc	128(%rdi), %xmm8
 6140        pxor	%xmm7, %xmm4
 6141        pshufd	$0x4e, %xmm0, %xmm5
 6142        pxor	%xmm0, %xmm5
 6143        movdqa	%xmm0, %xmm6
 6144        pclmulqdq	$0x11, %xmm7, %xmm6
 6145        aesenc	128(%rdi), %xmm9
 6146        aesenc	128(%rdi), %xmm10
 6147        pclmulqdq	$0x00, %xmm0, %xmm7
 6148        aesenc	128(%rdi), %xmm11
 6149        aesenc	128(%rdi), %xmm12
 6150        pclmulqdq	$0x00, %xmm5, %xmm4
 6151        aesenc	128(%rdi), %xmm13
 6152        aesenc	128(%rdi), %xmm14
 6153        aesenc	128(%rdi), %xmm15
 6154        pxor	%xmm7, %xmm1
 6155        pxor	%xmm7, %xmm2
 6156        pxor	%xmm6, %xmm1
 6157        pxor	%xmm6, %xmm3
 6158        pxor	%xmm4, %xmm1
 6159        movdqa	%xmm1, %xmm5
 6160        psrldq	$8, %xmm1
 6161        pslldq	$8, %xmm5
 6162        aesenc	144(%rdi), %xmm8
 6163        pxor	%xmm5, %xmm2
 6164        pxor	%xmm1, %xmm3
 6165        movdqa	%xmm2, %xmm7
 6166        movdqa	%xmm2, %xmm4
 6167        movdqa	%xmm2, %xmm5
 6168        aesenc	144(%rdi), %xmm9
 6169        pslld	$31, %xmm7
 6170        pslld	$30, %xmm4
 6171        pslld	$25, %xmm5
 6172        aesenc	144(%rdi), %xmm10
 6173        pxor	%xmm4, %xmm7
 6174        pxor	%xmm5, %xmm7
 6175        aesenc	144(%rdi), %xmm11
 6176        movdqa	%xmm7, %xmm4
 6177        pslldq	$12, %xmm7
 6178        psrldq	$4, %xmm4
 6179        aesenc	144(%rdi), %xmm12
 6180        pxor	%xmm7, %xmm2
 6181        movdqa	%xmm2, %xmm5
 6182        movdqa	%xmm2, %xmm1
 6183        movdqa	%xmm2, %xmm0
 6184        aesenc	144(%rdi), %xmm13
 6185        psrld	$0x01, %xmm5
 6186        psrld	$2, %xmm1
 6187        psrld	$7, %xmm0
 6188        aesenc	144(%rdi), %xmm14
 6189        pxor	%xmm1, %xmm5
 6190        pxor	%xmm0, %xmm5
 6191        aesenc	144(%rdi), %xmm15
 6192        pxor	%xmm4, %xmm5
 6193        pxor	%xmm5, %xmm2
 6194        pxor	%xmm3, %xmm2
 6195        cmpl	$11, %esi
 6196        movdqa	160(%rdi), %xmm7
 6197        jl	L_AES_GCM_decrypt_update_aesni_aesenc_128_ghash_avx_done
 6198        aesenc	%xmm7, %xmm8
 6199        aesenc	%xmm7, %xmm9
 6200        aesenc	%xmm7, %xmm10
 6201        aesenc	%xmm7, %xmm11
 6202        aesenc	%xmm7, %xmm12
 6203        aesenc	%xmm7, %xmm13
 6204        aesenc	%xmm7, %xmm14
 6205        aesenc	%xmm7, %xmm15
 6206        movdqa	176(%rdi), %xmm7
 6207        aesenc	%xmm7, %xmm8
 6208        aesenc	%xmm7, %xmm9
 6209        aesenc	%xmm7, %xmm10
 6210        aesenc	%xmm7, %xmm11
 6211        aesenc	%xmm7, %xmm12
 6212        aesenc	%xmm7, %xmm13
 6213        aesenc	%xmm7, %xmm14
 6214        aesenc	%xmm7, %xmm15
 6215        cmpl	$13, %esi
 6216        movdqa	192(%rdi), %xmm7
 6217        jl	L_AES_GCM_decrypt_update_aesni_aesenc_128_ghash_avx_done
 6218        aesenc	%xmm7, %xmm8
 6219        aesenc	%xmm7, %xmm9
 6220        aesenc	%xmm7, %xmm10
 6221        aesenc	%xmm7, %xmm11
 6222        aesenc	%xmm7, %xmm12
 6223        aesenc	%xmm7, %xmm13
 6224        aesenc	%xmm7, %xmm14
 6225        aesenc	%xmm7, %xmm15
 6226        movdqa	208(%rdi), %xmm7
 6227        aesenc	%xmm7, %xmm8
 6228        aesenc	%xmm7, %xmm9
 6229        aesenc	%xmm7, %xmm10
 6230        aesenc	%xmm7, %xmm11
 6231        aesenc	%xmm7, %xmm12
 6232        aesenc	%xmm7, %xmm13
 6233        aesenc	%xmm7, %xmm14
 6234        aesenc	%xmm7, %xmm15
 6235        movdqa	224(%rdi), %xmm7
 6236L_AES_GCM_decrypt_update_aesni_aesenc_128_ghash_avx_done:
 6237        aesenclast	%xmm7, %xmm8
 6238        aesenclast	%xmm7, %xmm9
 6239        movdqu	(%rcx), %xmm0
 6240        movdqu	16(%rcx), %xmm1
 6241        pxor	%xmm0, %xmm8
 6242        pxor	%xmm1, %xmm9
 6243        movdqu	%xmm8, (%rdx)
 6244        movdqu	%xmm9, 16(%rdx)
 6245        aesenclast	%xmm7, %xmm10
 6246        aesenclast	%xmm7, %xmm11
 6247        movdqu	32(%rcx), %xmm0
 6248        movdqu	48(%rcx), %xmm1
 6249        pxor	%xmm0, %xmm10
 6250        pxor	%xmm1, %xmm11
 6251        movdqu	%xmm10, 32(%rdx)
 6252        movdqu	%xmm11, 48(%rdx)
 6253        aesenclast	%xmm7, %xmm12
 6254        aesenclast	%xmm7, %xmm13
 6255        movdqu	64(%rcx), %xmm0
 6256        movdqu	80(%rcx), %xmm1
 6257        pxor	%xmm0, %xmm12
 6258        pxor	%xmm1, %xmm13
 6259        movdqu	%xmm12, 64(%rdx)
 6260        movdqu	%xmm13, 80(%rdx)
 6261        aesenclast	%xmm7, %xmm14
 6262        aesenclast	%xmm7, %xmm15
 6263        movdqu	96(%rcx), %xmm0
 6264        movdqu	112(%rcx), %xmm1
 6265        pxor	%xmm0, %xmm14
 6266        pxor	%xmm1, %xmm15
 6267        movdqu	%xmm14, 96(%rdx)
 6268        movdqu	%xmm15, 112(%rdx)
 6269        addl	$0x80, %r14d
 6270        cmpl	%r13d, %r14d
 6271        jl	L_AES_GCM_decrypt_update_aesni_ghash_128
 6272        movdqa	%xmm2, %xmm6
 6273        movdqu	(%rsp), %xmm5
 6274L_AES_GCM_decrypt_update_aesni_done_128:
 6275        movl	%r8d, %edx
 6276        cmpl	%edx, %r14d
 6277        jge	L_AES_GCM_decrypt_update_aesni_done_dec
 6278        movl	%r8d, %r13d
 6279        andl	$0xfffffff0, %r13d
 6280        cmpl	%r13d, %r14d
 6281        jge	L_AES_GCM_decrypt_update_aesni_last_block_done
 6282L_AES_GCM_decrypt_update_aesni_last_block_start:
 6283        leaq	(%r11,%r14,1), %rcx
 6284        leaq	(%r10,%r14,1), %rdx
 6285        movdqu	(%rcx), %xmm1
 6286        movdqa	%xmm5, %xmm0
 6287        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm1
 6288        pxor	%xmm6, %xmm1
 6289        movdqu	(%r12), %xmm8
 6290        movdqa	%xmm8, %xmm9
 6291        pshufb	L_aes_gcm_bswap_epi64(%rip), %xmm8
 6292        paddd	L_aes_gcm_one(%rip), %xmm9
 6293        pxor	(%rdi), %xmm8
 6294        movdqu	%xmm9, (%r12)
 6295        movdqa	%xmm1, %xmm10
 6296        pclmulqdq	$16, %xmm0, %xmm10
 6297        aesenc	16(%rdi), %xmm8
 6298        aesenc	32(%rdi), %xmm8
 6299        movdqa	%xmm1, %xmm11
 6300        pclmulqdq	$0x01, %xmm0, %xmm11
 6301        aesenc	48(%rdi), %xmm8
 6302        aesenc	64(%rdi), %xmm8
 6303        movdqa	%xmm1, %xmm12
 6304        pclmulqdq	$0x00, %xmm0, %xmm12
 6305        aesenc	80(%rdi), %xmm8
 6306        movdqa	%xmm1, %xmm1
 6307        pclmulqdq	$0x11, %xmm0, %xmm1
 6308        aesenc	96(%rdi), %xmm8
 6309        pxor	%xmm11, %xmm10
 6310        movdqa	%xmm10, %xmm2
 6311        psrldq	$8, %xmm10
 6312        pslldq	$8, %xmm2
 6313        aesenc	112(%rdi), %xmm8
 6314        movdqa	%xmm1, %xmm3
 6315        pxor	%xmm12, %xmm2
 6316        pxor	%xmm10, %xmm3
 6317        movdqa	L_aes_gcm_mod2_128(%rip), %xmm0
 6318        movdqa	%xmm2, %xmm11
 6319        pclmulqdq	$16, %xmm0, %xmm11
 6320        aesenc	128(%rdi), %xmm8
 6321        pshufd	$0x4e, %xmm2, %xmm10
 6322        pxor	%xmm11, %xmm10
 6323        movdqa	%xmm10, %xmm11
 6324        pclmulqdq	$16, %xmm0, %xmm11
 6325        aesenc	144(%rdi), %xmm8
 6326        pshufd	$0x4e, %xmm10, %xmm6
 6327        pxor	%xmm11, %xmm6
 6328        pxor	%xmm3, %xmm6
 6329        cmpl	$11, %esi
 6330        movdqa	160(%rdi), %xmm9
 6331        jl	L_AES_GCM_decrypt_update_aesni_aesenc_gfmul_last
 6332        aesenc	%xmm9, %xmm8
 6333        aesenc	176(%rdi), %xmm8
 6334        cmpl	$13, %esi
 6335        movdqa	192(%rdi), %xmm9
 6336        jl	L_AES_GCM_decrypt_update_aesni_aesenc_gfmul_last
 6337        aesenc	%xmm9, %xmm8
 6338        aesenc	208(%rdi), %xmm8
 6339        movdqa	224(%rdi), %xmm9
 6340L_AES_GCM_decrypt_update_aesni_aesenc_gfmul_last:
 6341        aesenclast	%xmm9, %xmm8
 6342        movdqu	(%rcx), %xmm9
 6343        pxor	%xmm9, %xmm8
 6344        movdqu	%xmm8, (%rdx)
 6345        addl	$16, %r14d
 6346        cmpl	%r13d, %r14d
 6347        jl	L_AES_GCM_decrypt_update_aesni_last_block_start
 6348L_AES_GCM_decrypt_update_aesni_last_block_done:
 6349L_AES_GCM_decrypt_update_aesni_done_dec:
 6350        movdqa	%xmm6, (%r9)
 6351        addq	$0xa8, %rsp
 6352        popq	%r15
 6353        popq	%r14
 6354        popq	%r12
 6355        popq	%r13
 6356        repz retq
 6357#ifndef __APPLE__
 6358.size	AES_GCM_decrypt_update_aesni,.-AES_GCM_decrypt_update_aesni
 6359#endif /* __APPLE__ */
 6360#ifndef __APPLE__
 6361.text
 6362.globl	AES_GCM_decrypt_final_aesni
 6363.type	AES_GCM_decrypt_final_aesni,@function
 6364.align	16
 6365AES_GCM_decrypt_final_aesni:
 6366#else
 6367.section	__TEXT,__text
 6368.globl	_AES_GCM_decrypt_final_aesni
 6369.p2align	4
 6370_AES_GCM_decrypt_final_aesni:
 6371#endif /* __APPLE__ */
 6372        pushq	%r13
 6373        pushq	%rbp
 6374        pushq	%r12
 6375        movl	%edx, %eax
 6376        movl	%ecx, %r10d
 6377        movl	%r8d, %r11d
 6378        movq	32(%rsp), %r8
 6379        movq	40(%rsp), %rbp
 6380        subq	$16, %rsp
 6381        movdqa	(%rdi), %xmm6
 6382        movdqa	(%r9), %xmm5
 6383        movdqa	(%r8), %xmm15
 6384        movdqa	%xmm5, %xmm8
 6385        movdqa	%xmm5, %xmm7
 6386        psrlq	$63, %xmm8
 6387        psllq	$0x01, %xmm7
 6388        pslldq	$8, %xmm8
 6389        por	%xmm8, %xmm7
 6390        pshufd	$0xff, %xmm5, %xmm5
 6391        psrad	$31, %xmm5
 6392        pand	L_aes_gcm_mod2_128(%rip), %xmm5
 6393        pxor	%xmm7, %xmm5
 6394        movl	%r10d, %edx
 6395        movl	%r11d, %ecx
 6396        shlq	$3, %rdx
 6397        shlq	$3, %rcx
 6398        pinsrq	$0x00, %rdx, %xmm0
 6399        pinsrq	$0x01, %rcx, %xmm0
 6400        pxor	%xmm0, %xmm6
 6401        pshufd	$0x4e, %xmm5, %xmm8
 6402        pshufd	$0x4e, %xmm6, %xmm9
 6403        movdqa	%xmm6, %xmm10
 6404        movdqa	%xmm6, %xmm7
 6405        pclmulqdq	$0x11, %xmm5, %xmm10
 6406        pclmulqdq	$0x00, %xmm5, %xmm7
 6407        pxor	%xmm5, %xmm8
 6408        pxor	%xmm6, %xmm9
 6409        pclmulqdq	$0x00, %xmm9, %xmm8
 6410        pxor	%xmm7, %xmm8
 6411        pxor	%xmm10, %xmm8
 6412        movdqa	%xmm8, %xmm9
 6413        movdqa	%xmm10, %xmm6
 6414        pslldq	$8, %xmm9
 6415        psrldq	$8, %xmm8
 6416        pxor	%xmm9, %xmm7
 6417        pxor	%xmm8, %xmm6
 6418        movdqa	%xmm7, %xmm11
 6419        movdqa	%xmm7, %xmm12
 6420        movdqa	%xmm7, %xmm13
 6421        pslld	$31, %xmm11
 6422        pslld	$30, %xmm12
 6423        pslld	$25, %xmm13
 6424        pxor	%xmm12, %xmm11
 6425        pxor	%xmm13, %xmm11
 6426        movdqa	%xmm11, %xmm12
 6427        psrldq	$4, %xmm12
 6428        pslldq	$12, %xmm11
 6429        pxor	%xmm11, %xmm7
 6430        movdqa	%xmm7, %xmm13
 6431        movdqa	%xmm7, %xmm9
 6432        movdqa	%xmm7, %xmm8
 6433        psrld	$0x01, %xmm13
 6434        psrld	$2, %xmm9
 6435        psrld	$7, %xmm8
 6436        pxor	%xmm9, %xmm13
 6437        pxor	%xmm8, %xmm13
 6438        pxor	%xmm12, %xmm13
 6439        pxor	%xmm7, %xmm13
 6440        pxor	%xmm13, %xmm6
 6441        pshufb	L_aes_gcm_bswap_mask(%rip), %xmm6
 6442        movdqu	%xmm15, %xmm0
 6443        pxor	%xmm6, %xmm0
 6444        cmpl	$16, %eax
 6445        je	L_AES_GCM_decrypt_final_aesni_cmp_tag_16
 6446        subq	$16, %rsp
 6447        xorq	%rcx, %rcx
 6448        xorq	%r12, %r12
 6449        movdqu	%xmm0, (%rsp)
 6450L_AES_GCM_decrypt_final_aesni_cmp_tag_loop:
 6451        movzbl	(%rsp,%rcx,1), %r13d
 6452        xorb	(%rsi,%rcx,1), %r13b
 6453        orb	%r13b, %r12b
 6454        incl	%ecx
 6455        cmpl	%eax, %ecx
 6456        jne	L_AES_GCM_decrypt_final_aesni_cmp_tag_loop
 6457        cmpb	$0x00, %r12b
 6458        sete	%r12b
 6459        addq	$16, %rsp
 6460        xorq	%rcx, %rcx
 6461        jmp	L_AES_GCM_decrypt_final_aesni_cmp_tag_done
 6462L_AES_GCM_decrypt_final_aesni_cmp_tag_16:
 6463        movdqu	(%rsi), %xmm1
 6464        pcmpeqb	%xmm1, %xmm0
 6465        pmovmskb	%xmm0, %rdx
 6466        # %%edx == 0xFFFF then return 1 else => return 0
 6467        xorl	%r12d, %r12d
 6468        cmpl	$0xffff, %edx
 6469        sete	%r12b
 6470L_AES_GCM_decrypt_final_aesni_cmp_tag_done:
 6471        movl	%r12d, (%rbp)
 6472        addq	$16, %rsp
 6473        popq	%r12
 6474        popq	%rbp
 6475        popq	%r13
 6476        repz retq
 6477#ifndef __APPLE__
 6478.size	AES_GCM_decrypt_final_aesni,.-AES_GCM_decrypt_final_aesni
 6479#endif /* __APPLE__ */
 6480#endif /* WOLFSSL_AESGCM_STREAM */
 6481#ifdef HAVE_INTEL_AVX1
 6482#ifndef __APPLE__
 6483.data
 6484#else
 6485.section	__DATA,__data
 6486#endif /* __APPLE__ */
 6487#ifndef __APPLE__
 6488.align	16
 6489#else
 6490.p2align	4
 6491#endif /* __APPLE__ */
 6492L_GCM_generate_m0_avx1_rev8:
 6493.quad	0x08090a0b0c0d0e0f,0x0001020304050607
 6494#ifndef __APPLE__
 6495.data
 6496#else
 6497.section	__DATA,__data
 6498#endif /* __APPLE__ */
 6499#ifndef __APPLE__
 6500.align	16
 6501#else
 6502.p2align	4
 6503#endif /* __APPLE__ */
 6504L_GCM_generate_m0_avx1_mod2_128:
 6505.quad	0x0000000000000000,0xe100000000000000
 6506#ifndef __APPLE__
 6507.text
 6508.globl	GCM_generate_m0_avx1
 6509.type	GCM_generate_m0_avx1,@function
 6510.align	16
 6511GCM_generate_m0_avx1:
 6512#else
 6513.section	__TEXT,__text
 6514.globl	_GCM_generate_m0_avx1
 6515.p2align	4
 6516_GCM_generate_m0_avx1:
 6517#endif /* __APPLE__ */
 6518        vmovdqu	L_GCM_generate_m0_avx1_rev8(%rip), %xmm9
 6519        vmovdqu	L_GCM_generate_m0_avx1_mod2_128(%rip), %xmm10
 6520        vpxor	%xmm8, %xmm8, %xmm8
 6521        vmovdqu	(%rdi), %xmm0
 6522        vmovdqu	%xmm8, (%rsi)
 6523        vmovdqu	%xmm0, %xmm8
 6524        vpshufb	%xmm9, %xmm0, %xmm0
 6525        vpsllq	$63, %xmm0, %xmm5
 6526        vpsrlq	$0x01, %xmm0, %xmm4
 6527        vpslldq	$8, %xmm5, %xmm1
 6528        vpsrldq	$8, %xmm5, %xmm5
 6529        vpshufd	$0xff, %xmm1, %xmm1
 6530        vpor	%xmm5, %xmm4, %xmm4
 6531        vpsrad	$31, %xmm1, %xmm1
 6532        vpand	%xmm10, %xmm1, %xmm1
 6533        vpxor	%xmm4, %xmm1, %xmm1
 6534        vpsllq	$63, %xmm1, %xmm5
 6535        vpsrlq	$0x01, %xmm1, %xmm4
 6536        vpslldq	$8, %xmm5, %xmm2
 6537        vpsrldq	$8, %xmm5, %xmm5
 6538        vpshufd	$0xff, %xmm2, %xmm2
 6539        vpor	%xmm5, %xmm4, %xmm4
 6540        vpsrad	$31, %xmm2, %xmm2
 6541        vpand	%xmm10, %xmm2, %xmm2
 6542        vpxor	%xmm4, %xmm2, %xmm2
 6543        vpsllq	$63, %xmm2, %xmm5
 6544        vpsrlq	$0x01, %xmm2, %xmm4
 6545        vpslldq	$8, %xmm5, %xmm3
 6546        vpsrldq	$8, %xmm5, %xmm5
 6547        vpshufd	$0xff, %xmm3, %xmm3
 6548        vpor	%xmm5, %xmm4, %xmm4
 6549        vpsrad	$31, %xmm3, %xmm3
 6550        vpand	%xmm10, %xmm3, %xmm3
 6551        vpxor	%xmm4, %xmm3, %xmm3
 6552        vpshufb	%xmm9, %xmm3, %xmm3
 6553        vpshufb	%xmm9, %xmm2, %xmm2
 6554        vpshufb	%xmm9, %xmm1, %xmm1
 6555        vpshufb	%xmm9, %xmm0, %xmm0
 6556        vpxor	%xmm2, %xmm3, %xmm8
 6557        vmovdqu	%xmm3, 16(%rsi)
 6558        vmovdqu	%xmm2, 32(%rsi)
 6559        vmovdqu	%xmm8, 48(%rsi)
 6560        vmovdqu	%xmm1, 64(%rsi)
 6561        vpxor	%xmm1, %xmm3, %xmm4
 6562        vpxor	%xmm1, %xmm2, %xmm5
 6563        vpxor	%xmm1, %xmm8, %xmm6
 6564        vmovdqu	%xmm4, 80(%rsi)
 6565        vmovdqu	%xmm5, 96(%rsi)
 6566        vmovdqu	%xmm6, 112(%rsi)
 6567        vmovdqu	%xmm0, 128(%rsi)
 6568        vpxor	%xmm0, %xmm1, %xmm1
 6569        vpxor	%xmm0, %xmm3, %xmm4
 6570        vpxor	%xmm0, %xmm2, %xmm6
 6571        vmovdqu	%xmm4, 144(%rsi)
 6572        vmovdqu	%xmm6, 160(%rsi)
 6573        vpxor	%xmm6, %xmm3, %xmm6
 6574        vmovdqu	%xmm6, 176(%rsi)
 6575        vmovdqu	%xmm1, 192(%rsi)
 6576        vpxor	%xmm1, %xmm3, %xmm4
 6577        vpxor	%xmm1, %xmm2, %xmm5
 6578        vpxor	%xmm1, %xmm8, %xmm6
 6579        vmovdqu	%xmm4, 208(%rsi)
 6580        vmovdqu	%xmm5, 224(%rsi)
 6581        vmovdqu	%xmm6, 240(%rsi)
 6582        vmovdqu	(%rsi), %xmm0
 6583        vmovdqu	16(%rsi), %xmm1
 6584        vmovdqu	32(%rsi), %xmm2
 6585        vmovdqu	48(%rsi), %xmm3
 6586        vpshufb	%xmm9, %xmm0, %xmm0
 6587        vpshufb	%xmm9, %xmm1, %xmm1
 6588        vpshufb	%xmm9, %xmm2, %xmm2
 6589        vpshufb	%xmm9, %xmm3, %xmm3
 6590        vpsllq	$60, %xmm0, %xmm4
 6591        vpsllq	$60, %xmm1, %xmm5
 6592        vpsllq	$60, %xmm2, %xmm6
 6593        vpsllq	$60, %xmm3, %xmm7
 6594        vpsrlq	$4, %xmm0, %xmm0
 6595        vpsrlq	$4, %xmm1, %xmm1
 6596        vpsrlq	$4, %xmm2, %xmm2
 6597        vpsrlq	$4, %xmm3, %xmm3
 6598        vpsrldq	$8, %xmm4, %xmm4
 6599        vpsrldq	$8, %xmm5, %xmm5
 6600        vpsrldq	$8, %xmm6, %xmm6
 6601        vpsrldq	$8, %xmm7, %xmm7
 6602        vpor	%xmm4, %xmm0, %xmm0
 6603        vpor	%xmm5, %xmm1, %xmm1
 6604        vpor	%xmm6, %xmm2, %xmm2
 6605        vpor	%xmm7, %xmm3, %xmm3
 6606        vpshufb	%xmm9, %xmm0, %xmm0
 6607        vpshufb	%xmm9, %xmm1, %xmm1
 6608        vpshufb	%xmm9, %xmm2, %xmm2
 6609        vpshufb	%xmm9, %xmm3, %xmm3
 6610        vmovdqu	%xmm0, 256(%rsi)
 6611        vmovdqu	%xmm1, 272(%rsi)
 6612        vmovdqu	%xmm2, 288(%rsi)
 6613        vmovdqu	%xmm3, 304(%rsi)
 6614        vmovdqu	64(%rsi), %xmm0
 6615        vmovdqu	80(%rsi), %xmm1
 6616        vmovdqu	96(%rsi), %xmm2
 6617        vmovdqu	112(%rsi), %xmm3
 6618        vpshufb	%xmm9, %xmm0, %xmm0
 6619        vpshufb	%xmm9, %xmm1, %xmm1
 6620        vpshufb	%xmm9, %xmm2, %xmm2
 6621        vpshufb	%xmm9, %xmm3, %xmm3
 6622        vpsllq	$60, %xmm0, %xmm4
 6623        vpsllq	$60, %xmm1, %xmm5
 6624        vpsllq	$60, %xmm2, %xmm6
 6625        vpsllq	$60, %xmm3, %xmm7
 6626        vpsrlq	$4, %xmm0, %xmm0
 6627        vpsrlq	$4, %xmm1, %xmm1
 6628        vpsrlq	$4, %xmm2, %xmm2
 6629        vpsrlq	$4, %xmm3, %xmm3
 6630        vpsrldq	$8, %xmm4, %xmm4
 6631        vpsrldq	$8, %xmm5, %xmm5
 6632        vpsrldq	$8, %xmm6, %xmm6
 6633        vpsrldq	$8, %xmm7, %xmm7
 6634        vpor	%xmm4, %xmm0, %xmm0
 6635        vpor	%xmm5, %xmm1, %xmm1
 6636        vpor	%xmm6, %xmm2, %xmm2
 6637        vpor	%xmm7, %xmm3, %xmm3
 6638        vpshufb	%xmm9, %xmm0, %xmm0
 6639        vpshufb	%xmm9, %xmm1, %xmm1
 6640        vpshufb	%xmm9, %xmm2, %xmm2
 6641        vpshufb	%xmm9, %xmm3, %xmm3
 6642        vmovdqu	%xmm0, 320(%rsi)
 6643        vmovdqu	%xmm1, 336(%rsi)
 6644        vmovdqu	%xmm2, 352(%rsi)
 6645        vmovdqu	%xmm3, 368(%rsi)
 6646        vmovdqu	128(%rsi), %xmm0
 6647        vmovdqu	144(%rsi), %xmm1
 6648        vmovdqu	160(%rsi), %xmm2
 6649        vmovdqu	176(%rsi), %xmm3
 6650        vpshufb	%xmm9, %xmm0, %xmm0
 6651        vpshufb	%xmm9, %xmm1, %xmm1
 6652        vpshufb	%xmm9, %xmm2, %xmm2
 6653        vpshufb	%xmm9, %xmm3, %xmm3
 6654        vpsllq	$60, %xmm0, %xmm4
 6655        vpsllq	$60, %xmm1, %xmm5
 6656        vpsllq	$60, %xmm2, %xmm6
 6657        vpsllq	$60, %xmm3, %xmm7
 6658        vpsrlq	$4, %xmm0, %xmm0
 6659        vpsrlq	$4, %xmm1, %xmm1
 6660        vpsrlq	$4, %xmm2, %xmm2
 6661        vpsrlq	$4, %xmm3, %xmm3
 6662        vpsrldq	$8, %xmm4, %xmm4
 6663        vpsrldq	$8, %xmm5, %xmm5
 6664        vpsrldq	$8, %xmm6, %xmm6
 6665        vpsrldq	$8, %xmm7, %xmm7
 6666        vpor	%xmm4, %xmm0, %xmm0
 6667        vpor	%xmm5, %xmm1, %xmm1
 6668        vpor	%xmm6, %xmm2, %xmm2
 6669        vpor	%xmm7, %xmm3, %xmm3
 6670        vpshufb	%xmm9, %xmm0, %xmm0
 6671        vpshufb	%xmm9, %xmm1, %xmm1
 6672        vpshufb	%xmm9, %xmm2, %xmm2
 6673        vpshufb	%xmm9, %xmm3, %xmm3
 6674        vmovdqu	%xmm0, 384(%rsi)
 6675        vmovdqu	%xmm1, 400(%rsi)
 6676        vmovdqu	%xmm2, 416(%rsi)
 6677        vmovdqu	%xmm3, 432(%rsi)
 6678        vmovdqu	192(%rsi), %xmm0
 6679        vmovdqu	208(%rsi), %xmm1
 6680        vmovdqu	224(%rsi), %xmm2
 6681        vmovdqu	240(%rsi), %xmm3
 6682        vpshufb	%xmm9, %xmm0, %xmm0
 6683        vpshufb	%xmm9, %xmm1, %xmm1
 6684        vpshufb	%xmm9, %xmm2, %xmm2
 6685        vpshufb	%xmm9, %xmm3, %xmm3
 6686        vpsllq	$60, %xmm0, %xmm4
 6687        vpsllq	$60, %xmm1, %xmm5
 6688        vpsllq	$60, %xmm2, %xmm6
 6689        vpsllq	$60, %xmm3, %xmm7
 6690        vpsrlq	$4, %xmm0, %xmm0
 6691        vpsrlq	$4, %xmm1, %xmm1
 6692        vpsrlq	$4, %xmm2, %xmm2
 6693        vpsrlq	$4, %xmm3, %xmm3
 6694        vpsrldq	$8, %xmm4, %xmm4
 6695        vpsrldq	$8, %xmm5, %xmm5
 6696        vpsrldq	$8, %xmm6, %xmm6
 6697        vpsrldq	$8, %xmm7, %xmm7
 6698        vpor	%xmm4, %xmm0, %xmm0
 6699        vpor	%xmm5, %xmm1, %xmm1
 6700        vpor	%xmm6, %xmm2, %xmm2
 6701        vpor	%xmm7, %xmm3, %xmm3
 6702        vpshufb	%xmm9, %xmm0, %xmm0
 6703        vpshufb	%xmm9, %xmm1, %xmm1
 6704        vpshufb	%xmm9, %xmm2, %xmm2
 6705        vpshufb	%xmm9, %xmm3, %xmm3
 6706        vmovdqu	%xmm0, 448(%rsi)
 6707        vmovdqu	%xmm1, 464(%rsi)
 6708        vmovdqu	%xmm2, 480(%rsi)
 6709        vmovdqu	%xmm3, 496(%rsi)
 6710        repz retq
 6711#ifndef __APPLE__
 6712.size	GCM_generate_m0_avx1,.-GCM_generate_m0_avx1
 6713#endif /* __APPLE__ */
 6714#ifndef __APPLE__
 6715.data
 6716#else
 6717.section	__DATA,__data
 6718#endif /* __APPLE__ */
 6719#ifndef __APPLE__
 6720.align	16
 6721#else
 6722.p2align	4
 6723#endif /* __APPLE__ */
 6724L_avx1_aes_gcm_one:
 6725.quad	0x0000000000000000,0x0000000000000001
 6726#ifndef __APPLE__
 6727.data
 6728#else
 6729.section	__DATA,__data
 6730#endif /* __APPLE__ */
 6731#ifndef __APPLE__
 6732.align	16
 6733#else
 6734.p2align	4
 6735#endif /* __APPLE__ */
 6736L_avx1_aes_gcm_two:
 6737.quad	0x0000000000000000,0x0000000000000002
 6738#ifndef __APPLE__
 6739.data
 6740#else
 6741.section	__DATA,__data
 6742#endif /* __APPLE__ */
 6743#ifndef __APPLE__
 6744.align	16
 6745#else
 6746.p2align	4
 6747#endif /* __APPLE__ */
 6748L_avx1_aes_gcm_three:
 6749.quad	0x0000000000000000,0x0000000000000003
 6750#ifndef __APPLE__
 6751.data
 6752#else
 6753.section	__DATA,__data
 6754#endif /* __APPLE__ */
 6755#ifndef __APPLE__
 6756.align	16
 6757#else
 6758.p2align	4
 6759#endif /* __APPLE__ */
 6760L_avx1_aes_gcm_four:
 6761.quad	0x0000000000000000,0x0000000000000004
 6762#ifndef __APPLE__
 6763.data
 6764#else
 6765.section	__DATA,__data
 6766#endif /* __APPLE__ */
 6767#ifndef __APPLE__
 6768.align	16
 6769#else
 6770.p2align	4
 6771#endif /* __APPLE__ */
 6772L_avx1_aes_gcm_five:
 6773.quad	0x0000000000000000,0x0000000000000005
 6774#ifndef __APPLE__
 6775.data
 6776#else
 6777.section	__DATA,__data
 6778#endif /* __APPLE__ */
 6779#ifndef __APPLE__
 6780.align	16
 6781#else
 6782.p2align	4
 6783#endif /* __APPLE__ */
 6784L_avx1_aes_gcm_six:
 6785.quad	0x0000000000000000,0x0000000000000006
 6786#ifndef __APPLE__
 6787.data
 6788#else
 6789.section	__DATA,__data
 6790#endif /* __APPLE__ */
 6791#ifndef __APPLE__
 6792.align	16
 6793#else
 6794.p2align	4
 6795#endif /* __APPLE__ */
 6796L_avx1_aes_gcm_seven:
 6797.quad	0x0000000000000000,0x0000000000000007
 6798#ifndef __APPLE__
 6799.data
 6800#else
 6801.section	__DATA,__data
 6802#endif /* __APPLE__ */
 6803#ifndef __APPLE__
 6804.align	16
 6805#else
 6806.p2align	4
 6807#endif /* __APPLE__ */
 6808L_avx1_aes_gcm_eight:
 6809.quad	0x0000000000000000,0x0000000000000008
 6810#ifndef __APPLE__
 6811.data
 6812#else
 6813.section	__DATA,__data
 6814#endif /* __APPLE__ */
 6815#ifndef __APPLE__
 6816.align	16
 6817#else
 6818.p2align	4
 6819#endif /* __APPLE__ */
 6820L_avx1_aes_gcm_bswap_epi64:
 6821.quad	0x0001020304050607,0x08090a0b0c0d0e0f
 6822#ifndef __APPLE__
 6823.data
 6824#else
 6825.section	__DATA,__data
 6826#endif /* __APPLE__ */
 6827#ifndef __APPLE__
 6828.align	16
 6829#else
 6830.p2align	4
 6831#endif /* __APPLE__ */
 6832L_avx1_aes_gcm_bswap_mask:
 6833.quad	0x08090a0b0c0d0e0f,0x0001020304050607
 6834#ifndef __APPLE__
 6835.data
 6836#else
 6837.section	__DATA,__data
 6838#endif /* __APPLE__ */
 6839#ifndef __APPLE__
 6840.align	16
 6841#else
 6842.p2align	4
 6843#endif /* __APPLE__ */
 6844L_avx1_aes_gcm_mod2_128:
 6845.quad	0x0000000000000001,0xc200000000000000
 6846#ifndef __APPLE__
 6847.text
 6848.globl	AES_GCM_encrypt_avx1
 6849.type	AES_GCM_encrypt_avx1,@function
 6850.align	16
 6851AES_GCM_encrypt_avx1:
 6852#else
 6853.section	__TEXT,__text
 6854.globl	_AES_GCM_encrypt_avx1
 6855.p2align	4
 6856_AES_GCM_encrypt_avx1:
 6857#endif /* __APPLE__ */
 6858        pushq	%r13
 6859        pushq	%r12
 6860        pushq	%rbx
 6861        pushq	%r14
 6862        pushq	%r15
 6863        movq	%rdx, %r12
 6864        movq	%rcx, %rax
 6865        movl	48(%rsp), %r11d
 6866        movl	56(%rsp), %ebx
 6867        movl	64(%rsp), %r14d
 6868        movq	72(%rsp), %r15
 6869        movl	80(%rsp), %r10d
 6870        subq	$0xa0, %rsp
 6871        vpxor	%xmm4, %xmm4, %xmm4
 6872        vpxor	%xmm6, %xmm6, %xmm6
 6873        movl	%ebx, %edx
 6874        cmpl	$12, %edx
 6875        jne	L_AES_GCM_encrypt_avx1_iv_not_12
 6876        # # Calculate values when IV is 12 bytes
 6877        # Set counter based on IV
 6878        movl	$0x1000000, %ecx
 6879        vmovq	(%rax), %xmm4
 6880        vpinsrd	$2, 8(%rax), %xmm4, %xmm4
 6881        vpinsrd	$3, %ecx, %xmm4, %xmm4
 6882        # H = Encrypt X(=0) and T = Encrypt counter
 6883        vmovdqa	(%r15), %xmm5
 6884        vpxor	%xmm5, %xmm4, %xmm1
 6885        vmovdqa	16(%r15), %xmm7
 6886        vaesenc	%xmm7, %xmm5, %xmm5
 6887        vaesenc	%xmm7, %xmm1, %xmm1
 6888        vmovdqa	32(%r15), %xmm7
 6889        vaesenc	%xmm7, %xmm5, %xmm5
 6890        vaesenc	%xmm7, %xmm1, %xmm1
 6891        vmovdqa	48(%r15), %xmm7
 6892        vaesenc	%xmm7, %xmm5, %xmm5
 6893        vaesenc	%xmm7, %xmm1, %xmm1
 6894        vmovdqa	64(%r15), %xmm7
 6895        vaesenc	%xmm7, %xmm5, %xmm5
 6896        vaesenc	%xmm7, %xmm1, %xmm1
 6897        vmovdqa	80(%r15), %xmm7
 6898        vaesenc	%xmm7, %xmm5, %xmm5
 6899        vaesenc	%xmm7, %xmm1, %xmm1
 6900        vmovdqa	96(%r15), %xmm7
 6901        vaesenc	%xmm7, %xmm5, %xmm5
 6902        vaesenc	%xmm7, %xmm1, %xmm1
 6903        vmovdqa	112(%r15), %xmm7
 6904        vaesenc	%xmm7, %xmm5, %xmm5
 6905        vaesenc	%xmm7, %xmm1, %xmm1
 6906        vmovdqa	128(%r15), %xmm7
 6907        vaesenc	%xmm7, %xmm5, %xmm5
 6908        vaesenc	%xmm7, %xmm1, %xmm1
 6909        vmovdqa	144(%r15), %xmm7
 6910        vaesenc	%xmm7, %xmm5, %xmm5
 6911        vaesenc	%xmm7, %xmm1, %xmm1
 6912        cmpl	$11, %r10d
 6913        vmovdqa	160(%r15), %xmm7
 6914        jl	L_AES_GCM_encrypt_avx1_calc_iv_12_last
 6915        vaesenc	%xmm7, %xmm5, %xmm5
 6916        vaesenc	%xmm7, %xmm1, %xmm1
 6917        vmovdqa	176(%r15), %xmm7
 6918        vaesenc	%xmm7, %xmm5, %xmm5
 6919        vaesenc	%xmm7, %xmm1, %xmm1
 6920        cmpl	$13, %r10d
 6921        vmovdqa	192(%r15), %xmm7
 6922        jl	L_AES_GCM_encrypt_avx1_calc_iv_12_last
 6923        vaesenc	%xmm7, %xmm5, %xmm5
 6924        vaesenc	%xmm7, %xmm1, %xmm1
 6925        vmovdqa	208(%r15), %xmm7
 6926        vaesenc	%xmm7, %xmm5, %xmm5
 6927        vaesenc	%xmm7, %xmm1, %xmm1
 6928        vmovdqa	224(%r15), %xmm7
 6929L_AES_GCM_encrypt_avx1_calc_iv_12_last:
 6930        vaesenclast	%xmm7, %xmm5, %xmm5
 6931        vaesenclast	%xmm7, %xmm1, %xmm1
 6932        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm5, %xmm5
 6933        vmovdqu	%xmm1, 144(%rsp)
 6934        jmp	L_AES_GCM_encrypt_avx1_iv_done
 6935L_AES_GCM_encrypt_avx1_iv_not_12:
 6936        # Calculate values when IV is not 12 bytes
 6937        # H = Encrypt X(=0)
 6938        vmovdqa	(%r15), %xmm5
 6939        vaesenc	16(%r15), %xmm5, %xmm5
 6940        vaesenc	32(%r15), %xmm5, %xmm5
 6941        vaesenc	48(%r15), %xmm5, %xmm5
 6942        vaesenc	64(%r15), %xmm5, %xmm5
 6943        vaesenc	80(%r15), %xmm5, %xmm5
 6944        vaesenc	96(%r15), %xmm5, %xmm5
 6945        vaesenc	112(%r15), %xmm5, %xmm5
 6946        vaesenc	128(%r15), %xmm5, %xmm5
 6947        vaesenc	144(%r15), %xmm5, %xmm5
 6948        cmpl	$11, %r10d
 6949        vmovdqa	160(%r15), %xmm9
 6950        jl	L_AES_GCM_encrypt_avx1_calc_iv_1_aesenc_avx_last
 6951        vaesenc	%xmm9, %xmm5, %xmm5
 6952        vaesenc	176(%r15), %xmm5, %xmm5
 6953        cmpl	$13, %r10d
 6954        vmovdqa	192(%r15), %xmm9
 6955        jl	L_AES_GCM_encrypt_avx1_calc_iv_1_aesenc_avx_last
 6956        vaesenc	%xmm9, %xmm5, %xmm5
 6957        vaesenc	208(%r15), %xmm5, %xmm5
 6958        vmovdqa	224(%r15), %xmm9
 6959L_AES_GCM_encrypt_avx1_calc_iv_1_aesenc_avx_last:
 6960        vaesenclast	%xmm9, %xmm5, %xmm5
 6961        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm5, %xmm5
 6962        # Calc counter
 6963        # Initialization vector
 6964        cmpl	$0x00, %edx
 6965        movq	$0x00, %rcx
 6966        je	L_AES_GCM_encrypt_avx1_calc_iv_done
 6967        cmpl	$16, %edx
 6968        jl	L_AES_GCM_encrypt_avx1_calc_iv_lt16
 6969        andl	$0xfffffff0, %edx
 6970L_AES_GCM_encrypt_avx1_calc_iv_16_loop:
 6971        vmovdqu	(%rax,%rcx,1), %xmm8
 6972        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm8, %xmm8
 6973        vpxor	%xmm8, %xmm4, %xmm4
 6974        # ghash_gfmul_avx
 6975        vpshufd	$0x4e, %xmm4, %xmm1
 6976        vpshufd	$0x4e, %xmm5, %xmm2
 6977        vpclmulqdq	$0x11, %xmm4, %xmm5, %xmm3
 6978        vpclmulqdq	$0x00, %xmm4, %xmm5, %xmm0
 6979        vpxor	%xmm4, %xmm1, %xmm1
 6980        vpxor	%xmm5, %xmm2, %xmm2
 6981        vpclmulqdq	$0x00, %xmm2, %xmm1, %xmm1
 6982        vpxor	%xmm0, %xmm1, %xmm1
 6983        vpxor	%xmm3, %xmm1, %xmm1
 6984        vmovdqa	%xmm0, %xmm7
 6985        vmovdqa	%xmm3, %xmm4
 6986        vpslldq	$8, %xmm1, %xmm2
 6987        vpsrldq	$8, %xmm1, %xmm1
 6988        vpxor	%xmm2, %xmm7, %xmm7
 6989        vpxor	%xmm1, %xmm4, %xmm4
 6990        vpsrld	$31, %xmm7, %xmm0
 6991        vpsrld	$31, %xmm4, %xmm1
 6992        vpslld	$0x01, %xmm7, %xmm7
 6993        vpslld	$0x01, %xmm4, %xmm4
 6994        vpsrldq	$12, %xmm0, %xmm2
 6995        vpslldq	$4, %xmm0, %xmm0
 6996        vpslldq	$4, %xmm1, %xmm1
 6997        vpor	%xmm2, %xmm4, %xmm4
 6998        vpor	%xmm0, %xmm7, %xmm7
 6999        vpor	%xmm1, %xmm4, %xmm4
 7000        vpslld	$31, %xmm7, %xmm0
 7001        vpslld	$30, %xmm7, %xmm1
 7002        vpslld	$25, %xmm7, %xmm2
 7003        vpxor	%xmm1, %xmm0, %xmm0
 7004        vpxor	%xmm2, %xmm0, %xmm0
 7005        vmovdqa	%xmm0, %xmm1
 7006        vpsrldq	$4, %xmm1, %xmm1
 7007        vpslldq	$12, %xmm0, %xmm0
 7008        vpxor	%xmm0, %xmm7, %xmm7
 7009        vpsrld	$0x01, %xmm7, %xmm2
 7010        vpsrld	$2, %xmm7, %xmm3
 7011        vpsrld	$7, %xmm7, %xmm0
 7012        vpxor	%xmm3, %xmm2, %xmm2
 7013        vpxor	%xmm0, %xmm2, %xmm2
 7014        vpxor	%xmm1, %xmm2, %xmm2
 7015        vpxor	%xmm7, %xmm2, %xmm2
 7016        vpxor	%xmm2, %xmm4, %xmm4
 7017        addl	$16, %ecx
 7018        cmpl	%edx, %ecx
 7019        jl	L_AES_GCM_encrypt_avx1_calc_iv_16_loop
 7020        movl	%ebx, %edx
 7021        cmpl	%edx, %ecx
 7022        je	L_AES_GCM_encrypt_avx1_calc_iv_done
 7023L_AES_GCM_encrypt_avx1_calc_iv_lt16:
 7024        subq	$16, %rsp
 7025        vpxor	%xmm8, %xmm8, %xmm8
 7026        xorl	%ebx, %ebx
 7027        vmovdqu	%xmm8, (%rsp)
 7028L_AES_GCM_encrypt_avx1_calc_iv_loop:
 7029        movzbl	(%rax,%rcx,1), %r13d
 7030        movb	%r13b, (%rsp,%rbx,1)
 7031        incl	%ecx
 7032        incl	%ebx
 7033        cmpl	%edx, %ecx
 7034        jl	L_AES_GCM_encrypt_avx1_calc_iv_loop
 7035        vmovdqu	(%rsp), %xmm8
 7036        addq	$16, %rsp
 7037        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm8, %xmm8
 7038        vpxor	%xmm8, %xmm4, %xmm4
 7039        # ghash_gfmul_avx
 7040        vpshufd	$0x4e, %xmm4, %xmm1
 7041        vpshufd	$0x4e, %xmm5, %xmm2
 7042        vpclmulqdq	$0x11, %xmm4, %xmm5, %xmm3
 7043        vpclmulqdq	$0x00, %xmm4, %xmm5, %xmm0
 7044        vpxor	%xmm4, %xmm1, %xmm1
 7045        vpxor	%xmm5, %xmm2, %xmm2
 7046        vpclmulqdq	$0x00, %xmm2, %xmm1, %xmm1
 7047        vpxor	%xmm0, %xmm1, %xmm1
 7048        vpxor	%xmm3, %xmm1, %xmm1
 7049        vmovdqa	%xmm0, %xmm7
 7050        vmovdqa	%xmm3, %xmm4
 7051        vpslldq	$8, %xmm1, %xmm2
 7052        vpsrldq	$8, %xmm1, %xmm1
 7053        vpxor	%xmm2, %xmm7, %xmm7
 7054        vpxor	%xmm1, %xmm4, %xmm4
 7055        vpsrld	$31, %xmm7, %xmm0
 7056        vpsrld	$31, %xmm4, %xmm1
 7057        vpslld	$0x01, %xmm7, %xmm7
 7058        vpslld	$0x01, %xmm4, %xmm4
 7059        vpsrldq	$12, %xmm0, %xmm2
 7060        vpslldq	$4, %xmm0, %xmm0
 7061        vpslldq	$4, %xmm1, %xmm1
 7062        vpor	%xmm2, %xmm4, %xmm4
 7063        vpor	%xmm0, %xmm7, %xmm7
 7064        vpor	%xmm1, %xmm4, %xmm4
 7065        vpslld	$31, %xmm7, %xmm0
 7066        vpslld	$30, %xmm7, %xmm1
 7067        vpslld	$25, %xmm7, %xmm2
 7068        vpxor	%xmm1, %xmm0, %xmm0
 7069        vpxor	%xmm2, %xmm0, %xmm0
 7070        vmovdqa	%xmm0, %xmm1
 7071        vpsrldq	$4, %xmm1, %xmm1
 7072        vpslldq	$12, %xmm0, %xmm0
 7073        vpxor	%xmm0, %xmm7, %xmm7
 7074        vpsrld	$0x01, %xmm7, %xmm2
 7075        vpsrld	$2, %xmm7, %xmm3
 7076        vpsrld	$7, %xmm7, %xmm0
 7077        vpxor	%xmm3, %xmm2, %xmm2
 7078        vpxor	%xmm0, %xmm2, %xmm2
 7079        vpxor	%xmm1, %xmm2, %xmm2
 7080        vpxor	%xmm7, %xmm2, %xmm2
 7081        vpxor	%xmm2, %xmm4, %xmm4
 7082L_AES_GCM_encrypt_avx1_calc_iv_done:
 7083        # T = Encrypt counter
 7084        vpxor	%xmm0, %xmm0, %xmm0
 7085        shll	$3, %edx
 7086        vmovq	%rdx, %xmm0
 7087        vpxor	%xmm0, %xmm4, %xmm4
 7088        # ghash_gfmul_avx
 7089        vpshufd	$0x4e, %xmm4, %xmm1
 7090        vpshufd	$0x4e, %xmm5, %xmm2
 7091        vpclmulqdq	$0x11, %xmm4, %xmm5, %xmm3
 7092        vpclmulqdq	$0x00, %xmm4, %xmm5, %xmm0
 7093        vpxor	%xmm4, %xmm1, %xmm1
 7094        vpxor	%xmm5, %xmm2, %xmm2
 7095        vpclmulqdq	$0x00, %xmm2, %xmm1, %xmm1
 7096        vpxor	%xmm0, %xmm1, %xmm1
 7097        vpxor	%xmm3, %xmm1, %xmm1
 7098        vmovdqa	%xmm0, %xmm7
 7099        vmovdqa	%xmm3, %xmm4
 7100        vpslldq	$8, %xmm1, %xmm2
 7101        vpsrldq	$8, %xmm1, %xmm1
 7102        vpxor	%xmm2, %xmm7, %xmm7
 7103        vpxor	%xmm1, %xmm4, %xmm4
 7104        vpsrld	$31, %xmm7, %xmm0
 7105        vpsrld	$31, %xmm4, %xmm1
 7106        vpslld	$0x01, %xmm7, %xmm7
 7107        vpslld	$0x01, %xmm4, %xmm4
 7108        vpsrldq	$12, %xmm0, %xmm2
 7109        vpslldq	$4, %xmm0, %xmm0
 7110        vpslldq	$4, %xmm1, %xmm1
 7111        vpor	%xmm2, %xmm4, %xmm4
 7112        vpor	%xmm0, %xmm7, %xmm7
 7113        vpor	%xmm1, %xmm4, %xmm4
 7114        vpslld	$31, %xmm7, %xmm0
 7115        vpslld	$30, %xmm7, %xmm1
 7116        vpslld	$25, %xmm7, %xmm2
 7117        vpxor	%xmm1, %xmm0, %xmm0
 7118        vpxor	%xmm2, %xmm0, %xmm0
 7119        vmovdqa	%xmm0, %xmm1
 7120        vpsrldq	$4, %xmm1, %xmm1
 7121        vpslldq	$12, %xmm0, %xmm0
 7122        vpxor	%xmm0, %xmm7, %xmm7
 7123        vpsrld	$0x01, %xmm7, %xmm2
 7124        vpsrld	$2, %xmm7, %xmm3
 7125        vpsrld	$7, %xmm7, %xmm0
 7126        vpxor	%xmm3, %xmm2, %xmm2
 7127        vpxor	%xmm0, %xmm2, %xmm2
 7128        vpxor	%xmm1, %xmm2, %xmm2
 7129        vpxor	%xmm7, %xmm2, %xmm2
 7130        vpxor	%xmm2, %xmm4, %xmm4
 7131        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm4, %xmm4
 7132        #   Encrypt counter
 7133        vmovdqa	(%r15), %xmm8
 7134        vpxor	%xmm4, %xmm8, %xmm8
 7135        vaesenc	16(%r15), %xmm8, %xmm8
 7136        vaesenc	32(%r15), %xmm8, %xmm8
 7137        vaesenc	48(%r15), %xmm8, %xmm8
 7138        vaesenc	64(%r15), %xmm8, %xmm8
 7139        vaesenc	80(%r15), %xmm8, %xmm8
 7140        vaesenc	96(%r15), %xmm8, %xmm8
 7141        vaesenc	112(%r15), %xmm8, %xmm8
 7142        vaesenc	128(%r15), %xmm8, %xmm8
 7143        vaesenc	144(%r15), %xmm8, %xmm8
 7144        cmpl	$11, %r10d
 7145        vmovdqa	160(%r15), %xmm9
 7146        jl	L_AES_GCM_encrypt_avx1_calc_iv_2_aesenc_avx_last
 7147        vaesenc	%xmm9, %xmm8, %xmm8
 7148        vaesenc	176(%r15), %xmm8, %xmm8
 7149        cmpl	$13, %r10d
 7150        vmovdqa	192(%r15), %xmm9
 7151        jl	L_AES_GCM_encrypt_avx1_calc_iv_2_aesenc_avx_last
 7152        vaesenc	%xmm9, %xmm8, %xmm8
 7153        vaesenc	208(%r15), %xmm8, %xmm8
 7154        vmovdqa	224(%r15), %xmm9
 7155L_AES_GCM_encrypt_avx1_calc_iv_2_aesenc_avx_last:
 7156        vaesenclast	%xmm9, %xmm8, %xmm8
 7157        vmovdqu	%xmm8, 144(%rsp)
 7158L_AES_GCM_encrypt_avx1_iv_done:
 7159        # Additional authentication data
 7160        movl	%r11d, %edx
 7161        cmpl	$0x00, %edx
 7162        je	L_AES_GCM_encrypt_avx1_calc_aad_done
 7163        xorl	%ecx, %ecx
 7164        cmpl	$16, %edx
 7165        jl	L_AES_GCM_encrypt_avx1_calc_aad_lt16
 7166        andl	$0xfffffff0, %edx
 7167L_AES_GCM_encrypt_avx1_calc_aad_16_loop:
 7168        vmovdqu	(%r12,%rcx,1), %xmm8
 7169        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm8, %xmm8
 7170        vpxor	%xmm8, %xmm6, %xmm6
 7171        # ghash_gfmul_avx
 7172        vpshufd	$0x4e, %xmm6, %xmm1
 7173        vpshufd	$0x4e, %xmm5, %xmm2
 7174        vpclmulqdq	$0x11, %xmm6, %xmm5, %xmm3
 7175        vpclmulqdq	$0x00, %xmm6, %xmm5, %xmm0
 7176        vpxor	%xmm6, %xmm1, %xmm1
 7177        vpxor	%xmm5, %xmm2, %xmm2
 7178        vpclmulqdq	$0x00, %xmm2, %xmm1, %xmm1
 7179        vpxor	%xmm0, %xmm1, %xmm1
 7180        vpxor	%xmm3, %xmm1, %xmm1
 7181        vmovdqa	%xmm0, %xmm7
 7182        vmovdqa	%xmm3, %xmm6
 7183        vpslldq	$8, %xmm1, %xmm2
 7184        vpsrldq	$8, %xmm1, %xmm1
 7185        vpxor	%xmm2, %xmm7, %xmm7
 7186        vpxor	%xmm1, %xmm6, %xmm6
 7187        vpsrld	$31, %xmm7, %xmm0
 7188        vpsrld	$31, %xmm6, %xmm1
 7189        vpslld	$0x01, %xmm7, %xmm7
 7190        vpslld	$0x01, %xmm6, %xmm6
 7191        vpsrldq	$12, %xmm0, %xmm2
 7192        vpslldq	$4, %xmm0, %xmm0
 7193        vpslldq	$4, %xmm1, %xmm1
 7194        vpor	%xmm2, %xmm6, %xmm6
 7195        vpor	%xmm0, %xmm7, %xmm7
 7196        vpor	%xmm1, %xmm6, %xmm6
 7197        vpslld	$31, %xmm7, %xmm0
 7198        vpslld	$30, %xmm7, %xmm1
 7199        vpslld	$25, %xmm7, %xmm2
 7200        vpxor	%xmm1, %xmm0, %xmm0
 7201        vpxor	%xmm2, %xmm0, %xmm0
 7202        vmovdqa	%xmm0, %xmm1
 7203        vpsrldq	$4, %xmm1, %xmm1
 7204        vpslldq	$12, %xmm0, %xmm0
 7205        vpxor	%xmm0, %xmm7, %xmm7
 7206        vpsrld	$0x01, %xmm7, %xmm2
 7207        vpsrld	$2, %xmm7, %xmm3
 7208        vpsrld	$7, %xmm7, %xmm0
 7209        vpxor	%xmm3, %xmm2, %xmm2
 7210        vpxor	%xmm0, %xmm2, %xmm2
 7211        vpxor	%xmm1, %xmm2, %xmm2
 7212        vpxor	%xmm7, %xmm2, %xmm2
 7213        vpxor	%xmm2, %xmm6, %xmm6
 7214        addl	$16, %ecx
 7215        cmpl	%edx, %ecx
 7216        jl	L_AES_GCM_encrypt_avx1_calc_aad_16_loop
 7217        movl	%r11d, %edx
 7218        cmpl	%edx, %ecx
 7219        je	L_AES_GCM_encrypt_avx1_calc_aad_done
 7220L_AES_GCM_encrypt_avx1_calc_aad_lt16:
 7221        subq	$16, %rsp
 7222        vpxor	%xmm8, %xmm8, %xmm8
 7223        xorl	%ebx, %ebx
 7224        vmovdqu	%xmm8, (%rsp)
 7225L_AES_GCM_encrypt_avx1_calc_aad_loop:
 7226        movzbl	(%r12,%rcx,1), %r13d
 7227        movb	%r13b, (%rsp,%rbx,1)
 7228        incl	%ecx
 7229        incl	%ebx
 7230        cmpl	%edx, %ecx
 7231        jl	L_AES_GCM_encrypt_avx1_calc_aad_loop
 7232        vmovdqu	(%rsp), %xmm8
 7233        addq	$16, %rsp
 7234        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm8, %xmm8
 7235        vpxor	%xmm8, %xmm6, %xmm6
 7236        # ghash_gfmul_avx
 7237        vpshufd	$0x4e, %xmm6, %xmm1
 7238        vpshufd	$0x4e, %xmm5, %xmm2
 7239        vpclmulqdq	$0x11, %xmm6, %xmm5, %xmm3
 7240        vpclmulqdq	$0x00, %xmm6, %xmm5, %xmm0
 7241        vpxor	%xmm6, %xmm1, %xmm1
 7242        vpxor	%xmm5, %xmm2, %xmm2
 7243        vpclmulqdq	$0x00, %xmm2, %xmm1, %xmm1
 7244        vpxor	%xmm0, %xmm1, %xmm1
 7245        vpxor	%xmm3, %xmm1, %xmm1
 7246        vmovdqa	%xmm0, %xmm7
 7247        vmovdqa	%xmm3, %xmm6
 7248        vpslldq	$8, %xmm1, %xmm2
 7249        vpsrldq	$8, %xmm1, %xmm1
 7250        vpxor	%xmm2, %xmm7, %xmm7
 7251        vpxor	%xmm1, %xmm6, %xmm6
 7252        vpsrld	$31, %xmm7, %xmm0
 7253        vpsrld	$31, %xmm6, %xmm1
 7254        vpslld	$0x01, %xmm7, %xmm7
 7255        vpslld	$0x01, %xmm6, %xmm6
 7256        vpsrldq	$12, %xmm0, %xmm2
 7257        vpslldq	$4, %xmm0, %xmm0
 7258        vpslldq	$4, %xmm1, %xmm1
 7259        vpor	%xmm2, %xmm6, %xmm6
 7260        vpor	%xmm0, %xmm7, %xmm7
 7261        vpor	%xmm1, %xmm6, %xmm6
 7262        vpslld	$31, %xmm7, %xmm0
 7263        vpslld	$30, %xmm7, %xmm1
 7264        vpslld	$25, %xmm7, %xmm2
 7265        vpxor	%xmm1, %xmm0, %xmm0
 7266        vpxor	%xmm2, %xmm0, %xmm0
 7267        vmovdqa	%xmm0, %xmm1
 7268        vpsrldq	$4, %xmm1, %xmm1
 7269        vpslldq	$12, %xmm0, %xmm0
 7270        vpxor	%xmm0, %xmm7, %xmm7
 7271        vpsrld	$0x01, %xmm7, %xmm2
 7272        vpsrld	$2, %xmm7, %xmm3
 7273        vpsrld	$7, %xmm7, %xmm0
 7274        vpxor	%xmm3, %xmm2, %xmm2
 7275        vpxor	%xmm0, %xmm2, %xmm2
 7276        vpxor	%xmm1, %xmm2, %xmm2
 7277        vpxor	%xmm7, %xmm2, %xmm2
 7278        vpxor	%xmm2, %xmm6, %xmm6
 7279L_AES_GCM_encrypt_avx1_calc_aad_done:
 7280        # Calculate counter and H
 7281        vpsrlq	$63, %xmm5, %xmm9
 7282        vpsllq	$0x01, %xmm5, %xmm8
 7283        vpslldq	$8, %xmm9, %xmm9
 7284        vpor	%xmm9, %xmm8, %xmm8
 7285        vpshufd	$0xff, %xmm5, %xmm5
 7286        vpsrad	$31, %xmm5, %xmm5
 7287        vpshufb	L_avx1_aes_gcm_bswap_epi64(%rip), %xmm4, %xmm4
 7288        vpand	L_avx1_aes_gcm_mod2_128(%rip), %xmm5, %xmm5
 7289        vpaddd	L_avx1_aes_gcm_one(%rip), %xmm4, %xmm4
 7290        vpxor	%xmm8, %xmm5, %xmm5
 7291        vmovdqu	%xmm4, 128(%rsp)
 7292        xorl	%ebx, %ebx
 7293        cmpl	$0x80, %r9d
 7294        movl	%r9d, %r13d
 7295        jl	L_AES_GCM_encrypt_avx1_done_128
 7296        andl	$0xffffff80, %r13d
 7297        vmovdqa	%xmm6, %xmm2
 7298        # H ^ 1
 7299        vmovdqu	%xmm5, (%rsp)
 7300        # H ^ 2
 7301        vpclmulqdq	$0x00, %xmm5, %xmm5, %xmm8
 7302        vpclmulqdq	$0x11, %xmm5, %xmm5, %xmm0
 7303        vpslld	$31, %xmm8, %xmm12
 7304        vpslld	$30, %xmm8, %xmm13
 7305        vpslld	$25, %xmm8, %xmm14
 7306        vpxor	%xmm13, %xmm12, %xmm12
 7307        vpxor	%xmm14, %xmm12, %xmm12
 7308        vpsrldq	$4, %xmm12, %xmm13
 7309        vpslldq	$12, %xmm12, %xmm12
 7310        vpxor	%xmm12, %xmm8, %xmm8
 7311        vpsrld	$0x01, %xmm8, %xmm14
 7312        vpsrld	$2, %xmm8, %xmm10
 7313        vpsrld	$7, %xmm8, %xmm9
 7314        vpxor	%xmm10, %xmm14, %xmm14
 7315        vpxor	%xmm9, %xmm14, %xmm14
 7316        vpxor	%xmm13, %xmm14, %xmm14
 7317        vpxor	%xmm8, %xmm14, %xmm14
 7318        vpxor	%xmm14, %xmm0, %xmm0
 7319        vmovdqu	%xmm0, 16(%rsp)
 7320        # H ^ 3
 7321        # ghash_gfmul_red_avx
 7322        vpshufd	$0x4e, %xmm5, %xmm9
 7323        vpshufd	$0x4e, %xmm0, %xmm10
 7324        vpclmulqdq	$0x11, %xmm5, %xmm0, %xmm11
 7325        vpclmulqdq	$0x00, %xmm5, %xmm0, %xmm8
 7326        vpxor	%xmm5, %xmm9, %xmm9
 7327        vpxor	%xmm0, %xmm10, %xmm10
 7328        vpclmulqdq	$0x00, %xmm10, %xmm9, %xmm9
 7329        vpxor	%xmm8, %xmm9, %xmm9
 7330        vpxor	%xmm11, %xmm9, %xmm9
 7331        vpslldq	$8, %xmm9, %xmm10
 7332        vpsrldq	$8, %xmm9, %xmm9
 7333        vpxor	%xmm10, %xmm8, %xmm8
 7334        vpxor	%xmm9, %xmm11, %xmm1
 7335        vpslld	$31, %xmm8, %xmm12
 7336        vpslld	$30, %xmm8, %xmm13
 7337        vpslld	$25, %xmm8, %xmm14
 7338        vpxor	%xmm13, %xmm12, %xmm12
 7339        vpxor	%xmm14, %xmm12, %xmm12
 7340        vpsrldq	$4, %xmm12, %xmm13
 7341        vpslldq	$12, %xmm12, %xmm12
 7342        vpxor	%xmm12, %xmm8, %xmm8
 7343        vpsrld	$0x01, %xmm8, %xmm14
 7344        vpsrld	$2, %xmm8, %xmm10
 7345        vpsrld	$7, %xmm8, %xmm9
 7346        vpxor	%xmm10, %xmm14, %xmm14
 7347        vpxor	%xmm9, %xmm14, %xmm14
 7348        vpxor	%xmm13, %xmm14, %xmm14
 7349        vpxor	%xmm8, %xmm14, %xmm14
 7350        vpxor	%xmm14, %xmm1, %xmm1
 7351        vmovdqu	%xmm1, 32(%rsp)
 7352        # H ^ 4
 7353        vpclmulqdq	$0x00, %xmm0, %xmm0, %xmm8
 7354        vpclmulqdq	$0x11, %xmm0, %xmm0, %xmm3
 7355        vpslld	$31, %xmm8, %xmm12
 7356        vpslld	$30, %xmm8, %xmm13
 7357        vpslld	$25, %xmm8, %xmm14
 7358        vpxor	%xmm13, %xmm12, %xmm12
 7359        vpxor	%xmm14, %xmm12, %xmm12
 7360        vpsrldq	$4, %xmm12, %xmm13
 7361        vpslldq	$12, %xmm12, %xmm12
 7362        vpxor	%xmm12, %xmm8, %xmm8
 7363        vpsrld	$0x01, %xmm8, %xmm14
 7364        vpsrld	$2, %xmm8, %xmm10
 7365        vpsrld	$7, %xmm8, %xmm9
 7366        vpxor	%xmm10, %xmm14, %xmm14
 7367        vpxor	%xmm9, %xmm14, %xmm14
 7368        vpxor	%xmm13, %xmm14, %xmm14
 7369        vpxor	%xmm8, %xmm14, %xmm14
 7370        vpxor	%xmm14, %xmm3, %xmm3
 7371        vmovdqu	%xmm3, 48(%rsp)
 7372        # H ^ 5
 7373        # ghash_gfmul_red_avx
 7374        vpshufd	$0x4e, %xmm0, %xmm9
 7375        vpshufd	$0x4e, %xmm1, %xmm10
 7376        vpclmulqdq	$0x11, %xmm0, %xmm1, %xmm11
 7377        vpclmulqdq	$0x00, %xmm0, %xmm1, %xmm8
 7378        vpxor	%xmm0, %xmm9, %xmm9
 7379        vpxor	%xmm1, %xmm10, %xmm10
 7380        vpclmulqdq	$0x00, %xmm10, %xmm9, %xmm9
 7381        vpxor	%xmm8, %xmm9, %xmm9
 7382        vpxor	%xmm11, %xmm9, %xmm9
 7383        vpslldq	$8, %xmm9, %xmm10
 7384        vpsrldq	$8, %xmm9, %xmm9
 7385        vpxor	%xmm10, %xmm8, %xmm8
 7386        vpxor	%xmm9, %xmm11, %xmm7
 7387        vpslld	$31, %xmm8, %xmm12
 7388        vpslld	$30, %xmm8, %xmm13
 7389        vpslld	$25, %xmm8, %xmm14
 7390        vpxor	%xmm13, %xmm12, %xmm12
 7391        vpxor	%xmm14, %xmm12, %xmm12
 7392        vpsrldq	$4, %xmm12, %xmm13
 7393        vpslldq	$12, %xmm12, %xmm12
 7394        vpxor	%xmm12, %xmm8, %xmm8
 7395        vpsrld	$0x01, %xmm8, %xmm14
 7396        vpsrld	$2, %xmm8, %xmm10
 7397        vpsrld	$7, %xmm8, %xmm9
 7398        vpxor	%xmm10, %xmm14, %xmm14
 7399        vpxor	%xmm9, %xmm14, %xmm14
 7400        vpxor	%xmm13, %xmm14, %xmm14
 7401        vpxor	%xmm8, %xmm14, %xmm14
 7402        vpxor	%xmm14, %xmm7, %xmm7
 7403        vmovdqu	%xmm7, 64(%rsp)
 7404        # H ^ 6
 7405        vpclmulqdq	$0x00, %xmm1, %xmm1, %xmm8
 7406        vpclmulqdq	$0x11, %xmm1, %xmm1, %xmm7
 7407        vpslld	$31, %xmm8, %xmm12
 7408        vpslld	$30, %xmm8, %xmm13
 7409        vpslld	$25, %xmm8, %xmm14
 7410        vpxor	%xmm13, %xmm12, %xmm12
 7411        vpxor	%xmm14, %xmm12, %xmm12
 7412        vpsrldq	$4, %xmm12, %xmm13
 7413        vpslldq	$12, %xmm12, %xmm12
 7414        vpxor	%xmm12, %xmm8, %xmm8
 7415        vpsrld	$0x01, %xmm8, %xmm14
 7416        vpsrld	$2, %xmm8, %xmm10
 7417        vpsrld	$7, %xmm8, %xmm9
 7418        vpxor	%xmm10, %xmm14, %xmm14
 7419        vpxor	%xmm9, %xmm14, %xmm14
 7420        vpxor	%xmm13, %xmm14, %xmm14
 7421        vpxor	%xmm8, %xmm14, %xmm14
 7422        vpxor	%xmm14, %xmm7, %xmm7
 7423        vmovdqu	%xmm7, 80(%rsp)
 7424        # H ^ 7
 7425        # ghash_gfmul_red_avx
 7426        vpshufd	$0x4e, %xmm1, %xmm9
 7427        vpshufd	$0x4e, %xmm3, %xmm10
 7428        vpclmulqdq	$0x11, %xmm1, %xmm3, %xmm11
 7429        vpclmulqdq	$0x00, %xmm1, %xmm3, %xmm8
 7430        vpxor	%xmm1, %xmm9, %xmm9
 7431        vpxor	%xmm3, %xmm10, %xmm10
 7432        vpclmulqdq	$0x00, %xmm10, %xmm9, %xmm9
 7433        vpxor	%xmm8, %xmm9, %xmm9
 7434        vpxor	%xmm11, %xmm9, %xmm9
 7435        vpslldq	$8, %xmm9, %xmm10
 7436        vpsrldq	$8, %xmm9, %xmm9
 7437        vpxor	%xmm10, %xmm8, %xmm8
 7438        vpxor	%xmm9, %xmm11, %xmm7
 7439        vpslld	$31, %xmm8, %xmm12
 7440        vpslld	$30, %xmm8, %xmm13
 7441        vpslld	$25, %xmm8, %xmm14
 7442        vpxor	%xmm13, %xmm12, %xmm12
 7443        vpxor	%xmm14, %xmm12, %xmm12
 7444        vpsrldq	$4, %xmm12, %xmm13
 7445        vpslldq	$12, %xmm12, %xmm12
 7446        vpxor	%xmm12, %xmm8, %xmm8
 7447        vpsrld	$0x01, %xmm8, %xmm14
 7448        vpsrld	$2, %xmm8, %xmm10
 7449        vpsrld	$7, %xmm8, %xmm9
 7450        vpxor	%xmm10, %xmm14, %xmm14
 7451        vpxor	%xmm9, %xmm14, %xmm14
 7452        vpxor	%xmm13, %xmm14, %xmm14
 7453        vpxor	%xmm8, %xmm14, %xmm14
 7454        vpxor	%xmm14, %xmm7, %xmm7
 7455        vmovdqu	%xmm7, 96(%rsp)
 7456        # H ^ 8
 7457        vpclmulqdq	$0x00, %xmm3, %xmm3, %xmm8
 7458        vpclmulqdq	$0x11, %xmm3, %xmm3, %xmm7
 7459        vpslld	$31, %xmm8, %xmm12
 7460        vpslld	$30, %xmm8, %xmm13
 7461        vpslld	$25, %xmm8, %xmm14
 7462        vpxor	%xmm13, %xmm12, %xmm12
 7463        vpxor	%xmm14, %xmm12, %xmm12
 7464        vpsrldq	$4, %xmm12, %xmm13
 7465        vpslldq	$12, %xmm12, %xmm12
 7466        vpxor	%xmm12, %xmm8, %xmm8
 7467        vpsrld	$0x01, %xmm8, %xmm14
 7468        vpsrld	$2, %xmm8, %xmm10
 7469        vpsrld	$7, %xmm8, %xmm9
 7470        vpxor	%xmm10, %xmm14, %xmm14
 7471        vpxor	%xmm9, %xmm14, %xmm14
 7472        vpxor	%xmm13, %xmm14, %xmm14
 7473        vpxor	%xmm8, %xmm14, %xmm14
 7474        vpxor	%xmm14, %xmm7, %xmm7
 7475        vmovdqu	%xmm7, 112(%rsp)
 7476        # First 128 bytes of input
 7477        vmovdqu	128(%rsp), %xmm0
 7478        vmovdqa	L_avx1_aes_gcm_bswap_epi64(%rip), %xmm1
 7479        vpshufb	%xmm1, %xmm0, %xmm8
 7480        vpaddd	L_avx1_aes_gcm_one(%rip), %xmm0, %xmm9
 7481        vpshufb	%xmm1, %xmm9, %xmm9
 7482        vpaddd	L_avx1_aes_gcm_two(%rip), %xmm0, %xmm10
 7483        vpshufb	%xmm1, %xmm10, %xmm10
 7484        vpaddd	L_avx1_aes_gcm_three(%rip), %xmm0, %xmm11
 7485        vpshufb	%xmm1, %xmm11, %xmm11
 7486        vpaddd	L_avx1_aes_gcm_four(%rip), %xmm0, %xmm12
 7487        vpshufb	%xmm1, %xmm12, %xmm12
 7488        vpaddd	L_avx1_aes_gcm_five(%rip), %xmm0, %xmm13
 7489        vpshufb	%xmm1, %xmm13, %xmm13
 7490        vpaddd	L_avx1_aes_gcm_six(%rip), %xmm0, %xmm14
 7491        vpshufb	%xmm1, %xmm14, %xmm14
 7492        vpaddd	L_avx1_aes_gcm_seven(%rip), %xmm0, %xmm15
 7493        vpshufb	%xmm1, %xmm15, %xmm15
 7494        vpaddd	L_avx1_aes_gcm_eight(%rip), %xmm0, %xmm0
 7495        vmovdqa	(%r15), %xmm7
 7496        vmovdqu	%xmm0, 128(%rsp)
 7497        vpxor	%xmm7, %xmm8, %xmm8
 7498        vpxor	%xmm7, %xmm9, %xmm9
 7499        vpxor	%xmm7, %xmm10, %xmm10
 7500        vpxor	%xmm7, %xmm11, %xmm11
 7501        vpxor	%xmm7, %xmm12, %xmm12
 7502        vpxor	%xmm7, %xmm13, %xmm13
 7503        vpxor	%xmm7, %xmm14, %xmm14
 7504        vpxor	%xmm7, %xmm15, %xmm15
 7505        vmovdqa	16(%r15), %xmm7
 7506        vaesenc	%xmm7, %xmm8, %xmm8
 7507        vaesenc	%xmm7, %xmm9, %xmm9
 7508        vaesenc	%xmm7, %xmm10, %xmm10
 7509        vaesenc	%xmm7, %xmm11, %xmm11
 7510        vaesenc	%xmm7, %xmm12, %xmm12
 7511        vaesenc	%xmm7, %xmm13, %xmm13
 7512        vaesenc	%xmm7, %xmm14, %xmm14
 7513        vaesenc	%xmm7, %xmm15, %xmm15
 7514        vmovdqa	32(%r15), %xmm7
 7515        vaesenc	%xmm7, %xmm8, %xmm8
 7516        vaesenc	%xmm7, %xmm9, %xmm9
 7517        vaesenc	%xmm7, %xmm10, %xmm10
 7518        vaesenc	%xmm7, %xmm11, %xmm11
 7519        vaesenc	%xmm7, %xmm12, %xmm12
 7520        vaesenc	%xmm7, %xmm13, %xmm13
 7521        vaesenc	%xmm7, %xmm14, %xmm14
 7522        vaesenc	%xmm7, %xmm15, %xmm15
 7523        vmovdqa	48(%r15), %xmm7
 7524        vaesenc	%xmm7, %xmm8, %xmm8
 7525        vaesenc	%xmm7, %xmm9, %xmm9
 7526        vaesenc	%xmm7, %xmm10, %xmm10
 7527        vaesenc	%xmm7, %xmm11, %xmm11
 7528        vaesenc	%xmm7, %xmm12, %xmm12
 7529        vaesenc	%xmm7, %xmm13, %xmm13
 7530        vaesenc	%xmm7, %xmm14, %xmm14
 7531        vaesenc	%xmm7, %xmm15, %xmm15
 7532        vmovdqa	64(%r15), %xmm7
 7533        vaesenc	%xmm7, %xmm8, %xmm8
 7534        vaesenc	%xmm7, %xmm9, %xmm9
 7535        vaesenc	%xmm7, %xmm10, %xmm10
 7536        vaesenc	%xmm7, %xmm11, %xmm11
 7537        vaesenc	%xmm7, %xmm12, %xmm12
 7538        vaesenc	%xmm7, %xmm13, %xmm13
 7539        vaesenc	%xmm7, %xmm14, %xmm14
 7540        vaesenc	%xmm7, %xmm15, %xmm15
 7541        vmovdqa	80(%r15), %xmm7
 7542        vaesenc	%xmm7, %xmm8, %xmm8
 7543        vaesenc	%xmm7, %xmm9, %xmm9
 7544        vaesenc	%xmm7, %xmm10, %xmm10
 7545        vaesenc	%xmm7, %xmm11, %xmm11
 7546        vaesenc	%xmm7, %xmm12, %xmm12
 7547        vaesenc	%xmm7, %xmm13, %xmm13
 7548        vaesenc	%xmm7, %xmm14, %xmm14
 7549        vaesenc	%xmm7, %xmm15, %xmm15
 7550        vmovdqa	96(%r15), %xmm7
 7551        vaesenc	%xmm7, %xmm8, %xmm8
 7552        vaesenc	%xmm7, %xmm9, %xmm9
 7553        vaesenc	%xmm7, %xmm10, %xmm10
 7554        vaesenc	%xmm7, %xmm11, %xmm11
 7555        vaesenc	%xmm7, %xmm12, %xmm12
 7556        vaesenc	%xmm7, %xmm13, %xmm13
 7557        vaesenc	%xmm7, %xmm14, %xmm14
 7558        vaesenc	%xmm7, %xmm15, %xmm15
 7559        vmovdqa	112(%r15), %xmm7
 7560        vaesenc	%xmm7, %xmm8, %xmm8
 7561        vaesenc	%xmm7, %xmm9, %xmm9
 7562        vaesenc	%xmm7, %xmm10, %xmm10
 7563        vaesenc	%xmm7, %xmm11, %xmm11
 7564        vaesenc	%xmm7, %xmm12, %xmm12
 7565        vaesenc	%xmm7, %xmm13, %xmm13
 7566        vaesenc	%xmm7, %xmm14, %xmm14
 7567        vaesenc	%xmm7, %xmm15, %xmm15
 7568        vmovdqa	128(%r15), %xmm7
 7569        vaesenc	%xmm7, %xmm8, %xmm8
 7570        vaesenc	%xmm7, %xmm9, %xmm9
 7571        vaesenc	%xmm7, %xmm10, %xmm10
 7572        vaesenc	%xmm7, %xmm11, %xmm11
 7573        vaesenc	%xmm7, %xmm12, %xmm12
 7574        vaesenc	%xmm7, %xmm13, %xmm13
 7575        vaesenc	%xmm7, %xmm14, %xmm14
 7576        vaesenc	%xmm7, %xmm15, %xmm15
 7577        vmovdqa	144(%r15), %xmm7
 7578        vaesenc	%xmm7, %xmm8, %xmm8
 7579        vaesenc	%xmm7, %xmm9, %xmm9
 7580        vaesenc	%xmm7, %xmm10, %xmm10
 7581        vaesenc	%xmm7, %xmm11, %xmm11
 7582        vaesenc	%xmm7, %xmm12, %xmm12
 7583        vaesenc	%xmm7, %xmm13, %xmm13
 7584        vaesenc	%xmm7, %xmm14, %xmm14
 7585        vaesenc	%xmm7, %xmm15, %xmm15
 7586        cmpl	$11, %r10d
 7587        vmovdqa	160(%r15), %xmm7
 7588        jl	L_AES_GCM_encrypt_avx1_aesenc_128_enc_done
 7589        vaesenc	%xmm7, %xmm8, %xmm8
 7590        vaesenc	%xmm7, %xmm9, %xmm9
 7591        vaesenc	%xmm7, %xmm10, %xmm10
 7592        vaesenc	%xmm7, %xmm11, %xmm11
 7593        vaesenc	%xmm7, %xmm12, %xmm12
 7594        vaesenc	%xmm7, %xmm13, %xmm13
 7595        vaesenc	%xmm7, %xmm14, %xmm14
 7596        vaesenc	%xmm7, %xmm15, %xmm15
 7597        vmovdqa	176(%r15), %xmm7
 7598        vaesenc	%xmm7, %xmm8, %xmm8
 7599        vaesenc	%xmm7, %xmm9, %xmm9
 7600        vaesenc	%xmm7, %xmm10, %xmm10
 7601        vaesenc	%xmm7, %xmm11, %xmm11
 7602        vaesenc	%xmm7, %xmm12, %xmm12
 7603        vaesenc	%xmm7, %xmm13, %xmm13
 7604        vaesenc	%xmm7, %xmm14, %xmm14
 7605        vaesenc	%xmm7, %xmm15, %xmm15
 7606        cmpl	$13, %r10d
 7607        vmovdqa	192(%r15), %xmm7
 7608        jl	L_AES_GCM_encrypt_avx1_aesenc_128_enc_done
 7609        vaesenc	%xmm7, %xmm8, %xmm8
 7610        vaesenc	%xmm7, %xmm9, %xmm9
 7611        vaesenc	%xmm7, %xmm10, %xmm10
 7612        vaesenc	%xmm7, %xmm11, %xmm11
 7613        vaesenc	%xmm7, %xmm12, %xmm12
 7614        vaesenc	%xmm7, %xmm13, %xmm13
 7615        vaesenc	%xmm7, %xmm14, %xmm14
 7616        vaesenc	%xmm7, %xmm15, %xmm15
 7617        vmovdqa	208(%r15), %xmm7
 7618        vaesenc	%xmm7, %xmm8, %xmm8
 7619        vaesenc	%xmm7, %xmm9, %xmm9
 7620        vaesenc	%xmm7, %xmm10, %xmm10
 7621        vaesenc	%xmm7, %xmm11, %xmm11
 7622        vaesenc	%xmm7, %xmm12, %xmm12
 7623        vaesenc	%xmm7, %xmm13, %xmm13
 7624        vaesenc	%xmm7, %xmm14, %xmm14
 7625        vaesenc	%xmm7, %xmm15, %xmm15
 7626        vmovdqa	224(%r15), %xmm7
 7627L_AES_GCM_encrypt_avx1_aesenc_128_enc_done:
 7628        vaesenclast	%xmm7, %xmm8, %xmm8
 7629        vaesenclast	%xmm7, %xmm9, %xmm9
 7630        vmovdqu	(%rdi), %xmm0
 7631        vmovdqu	16(%rdi), %xmm1
 7632        vpxor	%xmm0, %xmm8, %xmm8
 7633        vpxor	%xmm1, %xmm9, %xmm9
 7634        vmovdqu	%xmm8, (%rsi)
 7635        vmovdqu	%xmm9, 16(%rsi)
 7636        vaesenclast	%xmm7, %xmm10, %xmm10
 7637        vaesenclast	%xmm7, %xmm11, %xmm11
 7638        vmovdqu	32(%rdi), %xmm0
 7639        vmovdqu	48(%rdi), %xmm1
 7640        vpxor	%xmm0, %xmm10, %xmm10
 7641        vpxor	%xmm1, %xmm11, %xmm11
 7642        vmovdqu	%xmm10, 32(%rsi)
 7643        vmovdqu	%xmm11, 48(%rsi)
 7644        vaesenclast	%xmm7, %xmm12, %xmm12
 7645        vaesenclast	%xmm7, %xmm13, %xmm13
 7646        vmovdqu	64(%rdi), %xmm0
 7647        vmovdqu	80(%rdi), %xmm1
 7648        vpxor	%xmm0, %xmm12, %xmm12
 7649        vpxor	%xmm1, %xmm13, %xmm13
 7650        vmovdqu	%xmm12, 64(%rsi)
 7651        vmovdqu	%xmm13, 80(%rsi)
 7652        vaesenclast	%xmm7, %xmm14, %xmm14
 7653        vaesenclast	%xmm7, %xmm15, %xmm15
 7654        vmovdqu	96(%rdi), %xmm0
 7655        vmovdqu	112(%rdi), %xmm1
 7656        vpxor	%xmm0, %xmm14, %xmm14
 7657        vpxor	%xmm1, %xmm15, %xmm15
 7658        vmovdqu	%xmm14, 96(%rsi)
 7659        vmovdqu	%xmm15, 112(%rsi)
 7660        cmpl	$0x80, %r13d
 7661        movl	$0x80, %ebx
 7662        jle	L_AES_GCM_encrypt_avx1_end_128
 7663        # More 128 bytes of input
 7664L_AES_GCM_encrypt_avx1_ghash_128:
 7665        leaq	(%rdi,%rbx,1), %rcx
 7666        leaq	(%rsi,%rbx,1), %rdx
 7667        vmovdqu	128(%rsp), %xmm0
 7668        vmovdqa	L_avx1_aes_gcm_bswap_epi64(%rip), %xmm1
 7669        vpshufb	%xmm1, %xmm0, %xmm8
 7670        vpaddd	L_avx1_aes_gcm_one(%rip), %xmm0, %xmm9
 7671        vpshufb	%xmm1, %xmm9, %xmm9
 7672        vpaddd	L_avx1_aes_gcm_two(%rip), %xmm0, %xmm10
 7673        vpshufb	%xmm1, %xmm10, %xmm10
 7674        vpaddd	L_avx1_aes_gcm_three(%rip), %xmm0, %xmm11
 7675        vpshufb	%xmm1, %xmm11, %xmm11
 7676        vpaddd	L_avx1_aes_gcm_four(%rip), %xmm0, %xmm12
 7677        vpshufb	%xmm1, %xmm12, %xmm12
 7678        vpaddd	L_avx1_aes_gcm_five(%rip), %xmm0, %xmm13
 7679        vpshufb	%xmm1, %xmm13, %xmm13
 7680        vpaddd	L_avx1_aes_gcm_six(%rip), %xmm0, %xmm14
 7681        vpshufb	%xmm1, %xmm14, %xmm14
 7682        vpaddd	L_avx1_aes_gcm_seven(%rip), %xmm0, %xmm15
 7683        vpshufb	%xmm1, %xmm15, %xmm15
 7684        vpaddd	L_avx1_aes_gcm_eight(%rip), %xmm0, %xmm0
 7685        vmovdqa	(%r15), %xmm7
 7686        vmovdqu	%xmm0, 128(%rsp)
 7687        vpxor	%xmm7, %xmm8, %xmm8
 7688        vpxor	%xmm7, %xmm9, %xmm9
 7689        vpxor	%xmm7, %xmm10, %xmm10
 7690        vpxor	%xmm7, %xmm11, %xmm11
 7691        vpxor	%xmm7, %xmm12, %xmm12
 7692        vpxor	%xmm7, %xmm13, %xmm13
 7693        vpxor	%xmm7, %xmm14, %xmm14
 7694        vpxor	%xmm7, %xmm15, %xmm15
 7695        vmovdqu	112(%rsp), %xmm7
 7696        vmovdqu	-128(%rdx), %xmm0
 7697        vaesenc	16(%r15), %xmm8, %xmm8
 7698        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0
 7699        vpxor	%xmm2, %xmm0, %xmm0
 7700        vpshufd	$0x4e, %xmm7, %xmm1
 7701        vpshufd	$0x4e, %xmm0, %xmm5
 7702        vpxor	%xmm7, %xmm1, %xmm1
 7703        vpxor	%xmm0, %xmm5, %xmm5
 7704        vpclmulqdq	$0x11, %xmm7, %xmm0, %xmm3
 7705        vaesenc	16(%r15), %xmm9, %xmm9
 7706        vaesenc	16(%r15), %xmm10, %xmm10
 7707        vpclmulqdq	$0x00, %xmm7, %xmm0, %xmm2
 7708        vaesenc	16(%r15), %xmm11, %xmm11
 7709        vaesenc	16(%r15), %xmm12, %xmm12
 7710        vpclmulqdq	$0x00, %xmm5, %xmm1, %xmm1
 7711        vaesenc	16(%r15), %xmm13, %xmm13
 7712        vaesenc	16(%r15), %xmm14, %xmm14
 7713        vaesenc	16(%r15), %xmm15, %xmm15
 7714        vpxor	%xmm2, %xmm1, %xmm1
 7715        vpxor	%xmm3, %xmm1, %xmm1
 7716        vmovdqu	96(%rsp), %xmm7
 7717        vmovdqu	-112(%rdx), %xmm0
 7718        vpshufd	$0x4e, %xmm7, %xmm4
 7719        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0
 7720        vaesenc	32(%r15), %xmm8, %xmm8
 7721        vpxor	%xmm7, %xmm4, %xmm4
 7722        vpshufd	$0x4e, %xmm0, %xmm5
 7723        vpxor	%xmm0, %xmm5, %xmm5
 7724        vpclmulqdq	$0x11, %xmm7, %xmm0, %xmm6
 7725        vaesenc	32(%r15), %xmm9, %xmm9
 7726        vaesenc	32(%r15), %xmm10, %xmm10
 7727        vpclmulqdq	$0x00, %xmm7, %xmm0, %xmm7
 7728        vaesenc	32(%r15), %xmm11, %xmm11
 7729        vaesenc	32(%r15), %xmm12, %xmm12
 7730        vpclmulqdq	$0x00, %xmm5, %xmm4, %xmm4
 7731        vaesenc	32(%r15), %xmm13, %xmm13
 7732        vaesenc	32(%r15), %xmm14, %xmm14
 7733        vaesenc	32(%r15), %xmm15, %xmm15
 7734        vpxor	%xmm7, %xmm1, %xmm1
 7735        vpxor	%xmm7, %xmm2, %xmm2
 7736        vpxor	%xmm6, %xmm1, %xmm1
 7737        vpxor	%xmm6, %xmm3, %xmm3
 7738        vpxor	%xmm4, %xmm1, %xmm1
 7739        vmovdqu	80(%rsp), %xmm7
 7740        vmovdqu	-96(%rdx), %xmm0
 7741        vpshufd	$0x4e, %xmm7, %xmm4
 7742        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0
 7743        vaesenc	48(%r15), %xmm8, %xmm8
 7744        vpxor	%xmm7, %xmm4, %xmm4
 7745        vpshufd	$0x4e, %xmm0, %xmm5
 7746        vpxor	%xmm0, %xmm5, %xmm5
 7747        vpclmulqdq	$0x11, %xmm7, %xmm0, %xmm6
 7748        vaesenc	48(%r15), %xmm9, %xmm9
 7749        vaesenc	48(%r15), %xmm10, %xmm10
 7750        vpclmulqdq	$0x00, %xmm7, %xmm0, %xmm7
 7751        vaesenc	48(%r15), %xmm11, %xmm11
 7752        vaesenc	48(%r15), %xmm12, %xmm12
 7753        vpclmulqdq	$0x00, %xmm5, %xmm4, %xmm4
 7754        vaesenc	48(%r15), %xmm13, %xmm13
 7755        vaesenc	48(%r15), %xmm14, %xmm14
 7756        vaesenc	48(%r15), %xmm15, %xmm15
 7757        vpxor	%xmm7, %xmm1, %xmm1
 7758        vpxor	%xmm7, %xmm2, %xmm2
 7759        vpxor	%xmm6, %xmm1, %xmm1
 7760        vpxor	%xmm6, %xmm3, %xmm3
 7761        vpxor	%xmm4, %xmm1, %xmm1
 7762        vmovdqu	64(%rsp), %xmm7
 7763        vmovdqu	-80(%rdx), %xmm0
 7764        vpshufd	$0x4e, %xmm7, %xmm4
 7765        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0
 7766        vaesenc	64(%r15), %xmm8, %xmm8
 7767        vpxor	%xmm7, %xmm4, %xmm4
 7768        vpshufd	$0x4e, %xmm0, %xmm5
 7769        vpxor	%xmm0, %xmm5, %xmm5
 7770        vpclmulqdq	$0x11, %xmm7, %xmm0, %xmm6
 7771        vaesenc	64(%r15), %xmm9, %xmm9
 7772        vaesenc	64(%r15), %xmm10, %xmm10
 7773        vpclmulqdq	$0x00, %xmm7, %xmm0, %xmm7
 7774        vaesenc	64(%r15), %xmm11, %xmm11
 7775        vaesenc	64(%r15), %xmm12, %xmm12
 7776        vpclmulqdq	$0x00, %xmm5, %xmm4, %xmm4
 7777        vaesenc	64(%r15), %xmm13, %xmm13
 7778        vaesenc	64(%r15), %xmm14, %xmm14
 7779        vaesenc	64(%r15), %xmm15, %xmm15
 7780        vpxor	%xmm7, %xmm1, %xmm1
 7781        vpxor	%xmm7, %xmm2, %xmm2
 7782        vpxor	%xmm6, %xmm1, %xmm1
 7783        vpxor	%xmm6, %xmm3, %xmm3
 7784        vpxor	%xmm4, %xmm1, %xmm1
 7785        vmovdqu	48(%rsp), %xmm7
 7786        vmovdqu	-64(%rdx), %xmm0
 7787        vpshufd	$0x4e, %xmm7, %xmm4
 7788        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0
 7789        vaesenc	80(%r15), %xmm8, %xmm8
 7790        vpxor	%xmm7, %xmm4, %xmm4
 7791        vpshufd	$0x4e, %xmm0, %xmm5
 7792        vpxor	%xmm0, %xmm5, %xmm5
 7793        vpclmulqdq	$0x11, %xmm7, %xmm0, %xmm6
 7794        vaesenc	80(%r15), %xmm9, %xmm9
 7795        vaesenc	80(%r15), %xmm10, %xmm10
 7796        vpclmulqdq	$0x00, %xmm7, %xmm0, %xmm7
 7797        vaesenc	80(%r15), %xmm11, %xmm11
 7798        vaesenc	80(%r15), %xmm12, %xmm12
 7799        vpclmulqdq	$0x00, %xmm5, %xmm4, %xmm4
 7800        vaesenc	80(%r15), %xmm13, %xmm13
 7801        vaesenc	80(%r15), %xmm14, %xmm14
 7802        vaesenc	80(%r15), %xmm15, %xmm15
 7803        vpxor	%xmm7, %xmm1, %xmm1
 7804        vpxor	%xmm7, %xmm2, %xmm2
 7805        vpxor	%xmm6, %xmm1, %xmm1
 7806        vpxor	%xmm6, %xmm3, %xmm3
 7807        vpxor	%xmm4, %xmm1, %xmm1
 7808        vmovdqu	32(%rsp), %xmm7
 7809        vmovdqu	-48(%rdx), %xmm0
 7810        vpshufd	$0x4e, %xmm7, %xmm4
 7811        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0
 7812        vaesenc	96(%r15), %xmm8, %xmm8
 7813        vpxor	%xmm7, %xmm4, %xmm4
 7814        vpshufd	$0x4e, %xmm0, %xmm5
 7815        vpxor	%xmm0, %xmm5, %xmm5
 7816        vpclmulqdq	$0x11, %xmm7, %xmm0, %xmm6
 7817        vaesenc	96(%r15), %xmm9, %xmm9
 7818        vaesenc	96(%r15), %xmm10, %xmm10
 7819        vpclmulqdq	$0x00, %xmm7, %xmm0, %xmm7
 7820        vaesenc	96(%r15), %xmm11, %xmm11
 7821        vaesenc	96(%r15), %xmm12, %xmm12
 7822        vpclmulqdq	$0x00, %xmm5, %xmm4, %xmm4
 7823        vaesenc	96(%r15), %xmm13, %xmm13
 7824        vaesenc	96(%r15), %xmm14, %xmm14
 7825        vaesenc	96(%r15), %xmm15, %xmm15
 7826        vpxor	%xmm7, %xmm1, %xmm1
 7827        vpxor	%xmm7, %xmm2, %xmm2
 7828        vpxor	%xmm6, %xmm1, %xmm1
 7829        vpxor	%xmm6, %xmm3, %xmm3
 7830        vpxor	%xmm4, %xmm1, %xmm1
 7831        vmovdqu	16(%rsp), %xmm7
 7832        vmovdqu	-32(%rdx), %xmm0
 7833        vpshufd	$0x4e, %xmm7, %xmm4
 7834        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0
 7835        vaesenc	112(%r15), %xmm8, %xmm8
 7836        vpxor	%xmm7, %xmm4, %xmm4
 7837        vpshufd	$0x4e, %xmm0, %xmm5
 7838        vpxor	%xmm0, %xmm5, %xmm5
 7839        vpclmulqdq	$0x11, %xmm7, %xmm0, %xmm6
 7840        vaesenc	112(%r15), %xmm9, %xmm9
 7841        vaesenc	112(%r15), %xmm10, %xmm10
 7842        vpclmulqdq	$0x00, %xmm7, %xmm0, %xmm7
 7843        vaesenc	112(%r15), %xmm11, %xmm11
 7844        vaesenc	112(%r15), %xmm12, %xmm12
 7845        vpclmulqdq	$0x00, %xmm5, %xmm4, %xmm4
 7846        vaesenc	112(%r15), %xmm13, %xmm13
 7847        vaesenc	112(%r15), %xmm14, %xmm14
 7848        vaesenc	112(%r15), %xmm15, %xmm15
 7849        vpxor	%xmm7, %xmm1, %xmm1
 7850        vpxor	%xmm7, %xmm2, %xmm2
 7851        vpxor	%xmm6, %xmm1, %xmm1
 7852        vpxor	%xmm6, %xmm3, %xmm3
 7853        vpxor	%xmm4, %xmm1, %xmm1
 7854        vmovdqu	(%rsp), %xmm7
 7855        vmovdqu	-16(%rdx), %xmm0
 7856        vpshufd	$0x4e, %xmm7, %xmm4
 7857        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0
 7858        vaesenc	128(%r15), %xmm8, %xmm8
 7859        vpxor	%xmm7, %xmm4, %xmm4
 7860        vpshufd	$0x4e, %xmm0, %xmm5
 7861        vpxor	%xmm0, %xmm5, %xmm5
 7862        vpclmulqdq	$0x11, %xmm7, %xmm0, %xmm6
 7863        vaesenc	128(%r15), %xmm9, %xmm9
 7864        vaesenc	128(%r15), %xmm10, %xmm10
 7865        vpclmulqdq	$0x00, %xmm7, %xmm0, %xmm7
 7866        vaesenc	128(%r15), %xmm11, %xmm11
 7867        vaesenc	128(%r15), %xmm12, %xmm12
 7868        vpclmulqdq	$0x00, %xmm5, %xmm4, %xmm4
 7869        vaesenc	128(%r15), %xmm13, %xmm13
 7870        vaesenc	128(%r15), %xmm14, %xmm14
 7871        vaesenc	128(%r15), %xmm15, %xmm15
 7872        vpxor	%xmm7, %xmm1, %xmm1
 7873        vpxor	%xmm7, %xmm2, %xmm2
 7874        vpxor	%xmm6, %xmm1, %xmm1
 7875        vpxor	%xmm6, %xmm3, %xmm3
 7876        vpxor	%xmm4, %xmm1, %xmm1
 7877        vpslldq	$8, %xmm1, %xmm5
 7878        vpsrldq	$8, %xmm1, %xmm1
 7879        vaesenc	144(%r15), %xmm8, %xmm8
 7880        vpxor	%xmm5, %xmm2, %xmm2
 7881        vpxor	%xmm1, %xmm3, %xmm3
 7882        vaesenc	144(%r15), %xmm9, %xmm9
 7883        vpslld	$31, %xmm2, %xmm7
 7884        vpslld	$30, %xmm2, %xmm4
 7885        vpslld	$25, %xmm2, %xmm5
 7886        vaesenc	144(%r15), %xmm10, %xmm10
 7887        vpxor	%xmm4, %xmm7, %xmm7
 7888        vpxor	%xmm5, %xmm7, %xmm7
 7889        vaesenc	144(%r15), %xmm11, %xmm11
 7890        vpsrldq	$4, %xmm7, %xmm4
 7891        vpslldq	$12, %xmm7, %xmm7
 7892        vaesenc	144(%r15), %xmm12, %xmm12
 7893        vpxor	%xmm7, %xmm2, %xmm2
 7894        vpsrld	$0x01, %xmm2, %xmm5
 7895        vaesenc	144(%r15), %xmm13, %xmm13
 7896        vpsrld	$2, %xmm2, %xmm1
 7897        vpsrld	$7, %xmm2, %xmm0
 7898        vaesenc	144(%r15), %xmm14, %xmm14
 7899        vpxor	%xmm1, %xmm5, %xmm5
 7900        vpxor	%xmm0, %xmm5, %xmm5
 7901        vaesenc	144(%r15), %xmm15, %xmm15
 7902        vpxor	%xmm4, %xmm5, %xmm5
 7903        vpxor	%xmm5, %xmm2, %xmm2
 7904        vpxor	%xmm3, %xmm2, %xmm2
 7905        cmpl	$11, %r10d
 7906        vmovdqa	160(%r15), %xmm7
 7907        jl	L_AES_GCM_encrypt_avx1_aesenc_128_ghash_avx_done
 7908        vaesenc	%xmm7, %xmm8, %xmm8
 7909        vaesenc	%xmm7, %xmm9, %xmm9
 7910        vaesenc	%xmm7, %xmm10, %xmm10
 7911        vaesenc	%xmm7, %xmm11, %xmm11
 7912        vaesenc	%xmm7, %xmm12, %xmm12
 7913        vaesenc	%xmm7, %xmm13, %xmm13
 7914        vaesenc	%xmm7, %xmm14, %xmm14
 7915        vaesenc	%xmm7, %xmm15, %xmm15
 7916        vmovdqa	176(%r15), %xmm7
 7917        vaesenc	%xmm7, %xmm8, %xmm8
 7918        vaesenc	%xmm7, %xmm9, %xmm9
 7919        vaesenc	%xmm7, %xmm10, %xmm10
 7920        vaesenc	%xmm7, %xmm11, %xmm11
 7921        vaesenc	%xmm7, %xmm12, %xmm12
 7922        vaesenc	%xmm7, %xmm13, %xmm13
 7923        vaesenc	%xmm7, %xmm14, %xmm14
 7924        vaesenc	%xmm7, %xmm15, %xmm15
 7925        cmpl	$13, %r10d
 7926        vmovdqa	192(%r15), %xmm7
 7927        jl	L_AES_GCM_encrypt_avx1_aesenc_128_ghash_avx_done
 7928        vaesenc	%xmm7, %xmm8, %xmm8
 7929        vaesenc	%xmm7, %xmm9, %xmm9
 7930        vaesenc	%xmm7, %xmm10, %xmm10
 7931        vaesenc	%xmm7, %xmm11, %xmm11
 7932        vaesenc	%xmm7, %xmm12, %xmm12
 7933        vaesenc	%xmm7, %xmm13, %xmm13
 7934        vaesenc	%xmm7, %xmm14, %xmm14
 7935        vaesenc	%xmm7, %xmm15, %xmm15
 7936        vmovdqa	208(%r15), %xmm7
 7937        vaesenc	%xmm7, %xmm8, %xmm8
 7938        vaesenc	%xmm7, %xmm9, %xmm9
 7939        vaesenc	%xmm7, %xmm10, %xmm10
 7940        vaesenc	%xmm7, %xmm11, %xmm11
 7941        vaesenc	%xmm7, %xmm12, %xmm12
 7942        vaesenc	%xmm7, %xmm13, %xmm13
 7943        vaesenc	%xmm7, %xmm14, %xmm14
 7944        vaesenc	%xmm7, %xmm15, %xmm15
 7945        vmovdqa	224(%r15), %xmm7
 7946L_AES_GCM_encrypt_avx1_aesenc_128_ghash_avx_done:
 7947        vaesenclast	%xmm7, %xmm8, %xmm8
 7948        vaesenclast	%xmm7, %xmm9, %xmm9
 7949        vmovdqu	(%rcx), %xmm0
 7950        vmovdqu	16(%rcx), %xmm1
 7951        vpxor	%xmm0, %xmm8, %xmm8
 7952        vpxor	%xmm1, %xmm9, %xmm9
 7953        vmovdqu	%xmm8, (%rdx)
 7954        vmovdqu	%xmm9, 16(%rdx)
 7955        vaesenclast	%xmm7, %xmm10, %xmm10
 7956        vaesenclast	%xmm7, %xmm11, %xmm11
 7957        vmovdqu	32(%rcx), %xmm0
 7958        vmovdqu	48(%rcx), %xmm1
 7959        vpxor	%xmm0, %xmm10, %xmm10
 7960        vpxor	%xmm1, %xmm11, %xmm11
 7961        vmovdqu	%xmm10, 32(%rdx)
 7962        vmovdqu	%xmm11, 48(%rdx)
 7963        vaesenclast	%xmm7, %xmm12, %xmm12
 7964        vaesenclast	%xmm7, %xmm13, %xmm13
 7965        vmovdqu	64(%rcx), %xmm0
 7966        vmovdqu	80(%rcx), %xmm1
 7967        vpxor	%xmm0, %xmm12, %xmm12
 7968        vpxor	%xmm1, %xmm13, %xmm13
 7969        vmovdqu	%xmm12, 64(%rdx)
 7970        vmovdqu	%xmm13, 80(%rdx)
 7971        vaesenclast	%xmm7, %xmm14, %xmm14
 7972        vaesenclast	%xmm7, %xmm15, %xmm15
 7973        vmovdqu	96(%rcx), %xmm0
 7974        vmovdqu	112(%rcx), %xmm1
 7975        vpxor	%xmm0, %xmm14, %xmm14
 7976        vpxor	%xmm1, %xmm15, %xmm15
 7977        vmovdqu	%xmm14, 96(%rdx)
 7978        vmovdqu	%xmm15, 112(%rdx)
 7979        addl	$0x80, %ebx
 7980        cmpl	%r13d, %ebx
 7981        jl	L_AES_GCM_encrypt_avx1_ghash_128
 7982L_AES_GCM_encrypt_avx1_end_128:
 7983        vmovdqa	L_avx1_aes_gcm_bswap_mask(%rip), %xmm4
 7984        vpshufb	%xmm4, %xmm8, %xmm8
 7985        vpshufb	%xmm4, %xmm9, %xmm9
 7986        vpshufb	%xmm4, %xmm10, %xmm10
 7987        vpshufb	%xmm4, %xmm11, %xmm11
 7988        vpxor	%xmm2, %xmm8, %xmm8
 7989        vpshufb	%xmm4, %xmm12, %xmm12
 7990        vpshufb	%xmm4, %xmm13, %xmm13
 7991        vpshufb	%xmm4, %xmm14, %xmm14
 7992        vpshufb	%xmm4, %xmm15, %xmm15
 7993        vmovdqu	(%rsp), %xmm7
 7994        vmovdqu	16(%rsp), %xmm5
 7995        # ghash_gfmul_avx
 7996        vpshufd	$0x4e, %xmm15, %xmm1
 7997        vpshufd	$0x4e, %xmm7, %xmm2
 7998        vpclmulqdq	$0x11, %xmm15, %xmm7, %xmm3
 7999        vpclmulqdq	$0x00, %xmm15, %xmm7, %xmm0
 8000        vpxor	%xmm15, %xmm1, %xmm1
 8001        vpxor	%xmm7, %xmm2, %xmm2
 8002        vpclmulqdq	$0x00, %xmm2, %xmm1, %xmm1
 8003        vpxor	%xmm0, %xmm1, %xmm1
 8004        vpxor	%xmm3, %xmm1, %xmm1
 8005        vmovdqa	%xmm0, %xmm4
 8006        vmovdqa	%xmm3, %xmm6
 8007        vpslldq	$8, %xmm1, %xmm2
 8008        vpsrldq	$8, %xmm1, %xmm1
 8009        vpxor	%xmm2, %xmm4, %xmm4
 8010        vpxor	%xmm1, %xmm6, %xmm6
 8011        # ghash_gfmul_xor_avx
 8012        vpshufd	$0x4e, %xmm14, %xmm1
 8013        vpshufd	$0x4e, %xmm5, %xmm2
 8014        vpclmulqdq	$0x11, %xmm14, %xmm5, %xmm3
 8015        vpclmulqdq	$0x00, %xmm14, %xmm5, %xmm0
 8016        vpxor	%xmm14, %xmm1, %xmm1
 8017        vpxor	%xmm5, %xmm2, %xmm2
 8018        vpclmulqdq	$0x00, %xmm2, %xmm1, %xmm1
 8019        vpxor	%xmm0, %xmm1, %xmm1
 8020        vpxor	%xmm3, %xmm1, %xmm1
 8021        vpxor	%xmm0, %xmm4, %xmm4
 8022        vpxor	%xmm3, %xmm6, %xmm6
 8023        vpslldq	$8, %xmm1, %xmm2
 8024        vpsrldq	$8, %xmm1, %xmm1
 8025        vpxor	%xmm2, %xmm4, %xmm4
 8026        vpxor	%xmm1, %xmm6, %xmm6
 8027        vmovdqu	32(%rsp), %xmm7
 8028        vmovdqu	48(%rsp), %xmm5
 8029        # ghash_gfmul_xor_avx
 8030        vpshufd	$0x4e, %xmm13, %xmm1
 8031        vpshufd	$0x4e, %xmm7, %xmm2
 8032        vpclmulqdq	$0x11, %xmm13, %xmm7, %xmm3
 8033        vpclmulqdq	$0x00, %xmm13, %xmm7, %xmm0
 8034        vpxor	%xmm13, %xmm1, %xmm1
 8035        vpxor	%xmm7, %xmm2, %xmm2
 8036        vpclmulqdq	$0x00, %xmm2, %xmm1, %xmm1
 8037        vpxor	%xmm0, %xmm1, %xmm1
 8038        vpxor	%xmm3, %xmm1, %xmm1
 8039        vpxor	%xmm0, %xmm4, %xmm4
 8040        vpxor	%xmm3, %xmm6, %xmm6
 8041        vpslldq	$8, %xmm1, %xmm2
 8042        vpsrldq	$8, %xmm1, %xmm1
 8043        vpxor	%xmm2, %xmm4, %xmm4
 8044        vpxor	%xmm1, %xmm6, %xmm6
 8045        # ghash_gfmul_xor_avx
 8046        vpshufd	$0x4e, %xmm12, %xmm1
 8047        vpshufd	$0x4e, %xmm5, %xmm2
 8048        vpclmulqdq	$0x11, %xmm12, %xmm5, %xmm3
 8049        vpclmulqdq	$0x00, %xmm12, %xmm5, %xmm0
 8050        vpxor	%xmm12, %xmm1, %xmm1
 8051        vpxor	%xmm5, %xmm2, %xmm2
 8052        vpclmulqdq	$0x00, %xmm2, %xmm1, %xmm1
 8053        vpxor	%xmm0, %xmm1, %xmm1
 8054        vpxor	%xmm3, %xmm1, %xmm1
 8055        vpxor	%xmm0, %xmm4, %xmm4
 8056        vpxor	%xmm3, %xmm6, %xmm6
 8057        vpslldq	$8, %xmm1, %xmm2
 8058        vpsrldq	$8, %xmm1, %xmm1
 8059        vpxor	%xmm2, %xmm4, %xmm4
 8060        vpxor	%xmm1, %xmm6, %xmm6
 8061        vmovdqu	64(%rsp), %xmm7
 8062        vmovdqu	80(%rsp), %xmm5
 8063        # ghash_gfmul_xor_avx
 8064        vpshufd	$0x4e, %xmm11, %xmm1
 8065        vpshufd	$0x4e, %xmm7, %xmm2
 8066        vpclmulqdq	$0x11, %xmm11, %xmm7, %xmm3
 8067        vpclmulqdq	$0x00, %xmm11, %xmm7, %xmm0
 8068        vpxor	%xmm11, %xmm1, %xmm1
 8069        vpxor	%xmm7, %xmm2, %xmm2
 8070        vpclmulqdq	$0x00, %xmm2, %xmm1, %xmm1
 8071        vpxor	%xmm0, %xmm1, %xmm1
 8072        vpxor	%xmm3, %xmm1, %xmm1
 8073        vpxor	%xmm0, %xmm4, %xmm4
 8074        vpxor	%xmm3, %xmm6, %xmm6
 8075        vpslldq	$8, %xmm1, %xmm2
 8076        vpsrldq	$8, %xmm1, %xmm1
 8077        vpxor	%xmm2, %xmm4, %xmm4
 8078        vpxor	%xmm1, %xmm6, %xmm6
 8079        # ghash_gfmul_xor_avx
 8080        vpshufd	$0x4e, %xmm10, %xmm1
 8081        vpshufd	$0x4e, %xmm5, %xmm2
 8082        vpclmulqdq	$0x11, %xmm10, %xmm5, %xmm3
 8083        vpclmulqdq	$0x00, %xmm10, %xmm5, %xmm0
 8084        vpxor	%xmm10, %xmm1, %xmm1
 8085        vpxor	%xmm5, %xmm2, %xmm2
 8086        vpclmulqdq	$0x00, %xmm2, %xmm1, %xmm1
 8087        vpxor	%xmm0, %xmm1, %xmm1
 8088        vpxor	%xmm3, %xmm1, %xmm1
 8089        vpxor	%xmm0, %xmm4, %xmm4
 8090        vpxor	%xmm3, %xmm6, %xmm6
 8091        vpslldq	$8, %xmm1, %xmm2
 8092        vpsrldq	$8, %xmm1, %xmm1
 8093        vpxor	%xmm2, %xmm4, %xmm4
 8094        vpxor	%xmm1, %xmm6, %xmm6
 8095        vmovdqu	96(%rsp), %xmm7
 8096        vmovdqu	112(%rsp), %xmm5
 8097        # ghash_gfmul_xor_avx
 8098        vpshufd	$0x4e, %xmm9, %xmm1
 8099        vpshufd	$0x4e, %xmm7, %xmm2
 8100        vpclmulqdq	$0x11, %xmm9, %xmm7, %xmm3
 8101        vpclmulqdq	$0x00, %xmm9, %xmm7, %xmm0
 8102        vpxor	%xmm9, %xmm1, %xmm1
 8103        vpxor	%xmm7, %xmm2, %xmm2
 8104        vpclmulqdq	$0x00, %xmm2, %xmm1, %xmm1
 8105        vpxor	%xmm0, %xmm1, %xmm1
 8106        vpxor	%xmm3, %xmm1, %xmm1
 8107        vpxor	%xmm0, %xmm4, %xmm4
 8108        vpxor	%xmm3, %xmm6, %xmm6
 8109        vpslldq	$8, %xmm1, %xmm2
 8110        vpsrldq	$8, %xmm1, %xmm1
 8111        vpxor	%xmm2, %xmm4, %xmm4
 8112        vpxor	%xmm1, %xmm6, %xmm6
 8113        # ghash_gfmul_xor_avx
 8114        vpshufd	$0x4e, %xmm8, %xmm1
 8115        vpshufd	$0x4e, %xmm5, %xmm2
 8116        vpclmulqdq	$0x11, %xmm8, %xmm5, %xmm3
 8117        vpclmulqdq	$0x00, %xmm8, %xmm5, %xmm0
 8118        vpxor	%xmm8, %xmm1, %xmm1
 8119        vpxor	%xmm5, %xmm2, %xmm2
 8120        vpclmulqdq	$0x00, %xmm2, %xmm1, %xmm1
 8121        vpxor	%xmm0, %xmm1, %xmm1
 8122        vpxor	%xmm3, %xmm1, %xmm1
 8123        vpxor	%xmm0, %xmm4, %xmm4
 8124        vpxor	%xmm3, %xmm6, %xmm6
 8125        vpslldq	$8, %xmm1, %xmm2
 8126        vpsrldq	$8, %xmm1, %xmm1
 8127        vpxor	%xmm2, %xmm4, %xmm4
 8128        vpxor	%xmm1, %xmm6, %xmm6
 8129        vpslld	$31, %xmm4, %xmm0
 8130        vpslld	$30, %xmm4, %xmm1
 8131        vpslld	$25, %xmm4, %xmm2
 8132        vpxor	%xmm1, %xmm0, %xmm0
 8133        vpxor	%xmm2, %xmm0, %xmm0
 8134        vmovdqa	%xmm0, %xmm1
 8135        vpsrldq	$4, %xmm1, %xmm1
 8136        vpslldq	$12, %xmm0, %xmm0
 8137        vpxor	%xmm0, %xmm4, %xmm4
 8138        vpsrld	$0x01, %xmm4, %xmm2
 8139        vpsrld	$2, %xmm4, %xmm3
 8140        vpsrld	$7, %xmm4, %xmm0
 8141        vpxor	%xmm3, %xmm2, %xmm2
 8142        vpxor	%xmm0, %xmm2, %xmm2
 8143        vpxor	%xmm1, %xmm2, %xmm2
 8144        vpxor	%xmm4, %xmm2, %xmm2
 8145        vpxor	%xmm2, %xmm6, %xmm6
 8146        vmovdqu	(%rsp), %xmm5
 8147L_AES_GCM_encrypt_avx1_done_128:
 8148        movl	%r9d, %edx
 8149        cmpl	%edx, %ebx
 8150        jge	L_AES_GCM_encrypt_avx1_done_enc
 8151        movl	%r9d, %r13d
 8152        andl	$0xfffffff0, %r13d
 8153        cmpl	%r13d, %ebx
 8154        jge	L_AES_GCM_encrypt_avx1_last_block_done
 8155        vmovdqu	128(%rsp), %xmm9
 8156        vpshufb	L_avx1_aes_gcm_bswap_epi64(%rip), %xmm9, %xmm8
 8157        vpaddd	L_avx1_aes_gcm_one(%rip), %xmm9, %xmm9
 8158        vmovdqu	%xmm9, 128(%rsp)
 8159        vpxor	(%r15), %xmm8, %xmm8
 8160        vaesenc	16(%r15), %xmm8, %xmm8
 8161        vaesenc	32(%r15), %xmm8, %xmm8
 8162        vaesenc	48(%r15), %xmm8, %xmm8
 8163        vaesenc	64(%r15), %xmm8, %xmm8
 8164        vaesenc	80(%r15), %xmm8, %xmm8
 8165        vaesenc	96(%r15), %xmm8, %xmm8
 8166        vaesenc	112(%r15), %xmm8, %xmm8
 8167        vaesenc	128(%r15), %xmm8, %xmm8
 8168        vaesenc	144(%r15), %xmm8, %xmm8
 8169        cmpl	$11, %r10d
 8170        vmovdqa	160(%r15), %xmm9
 8171        jl	L_AES_GCM_encrypt_avx1_aesenc_block_last
 8172        vaesenc	%xmm9, %xmm8, %xmm8
 8173        vaesenc	176(%r15), %xmm8, %xmm8
 8174        cmpl	$13, %r10d
 8175        vmovdqa	192(%r15), %xmm9
 8176        jl	L_AES_GCM_encrypt_avx1_aesenc_block_last
 8177        vaesenc	%xmm9, %xmm8, %xmm8
 8178        vaesenc	208(%r15), %xmm8, %xmm8
 8179        vmovdqa	224(%r15), %xmm9
 8180L_AES_GCM_encrypt_avx1_aesenc_block_last:
 8181        vaesenclast	%xmm9, %xmm8, %xmm8
 8182        vmovdqu	(%rdi,%rbx,1), %xmm9
 8183        vpxor	%xmm9, %xmm8, %xmm8
 8184        vmovdqu	%xmm8, (%rsi,%rbx,1)
 8185        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm8, %xmm8
 8186        vpxor	%xmm8, %xmm6, %xmm6
 8187        addl	$16, %ebx
 8188        cmpl	%r13d, %ebx
 8189        jge	L_AES_GCM_encrypt_avx1_last_block_ghash
 8190L_AES_GCM_encrypt_avx1_last_block_start:
 8191        vmovdqu	(%rdi,%rbx,1), %xmm13
 8192        vmovdqu	128(%rsp), %xmm9
 8193        vpshufb	L_avx1_aes_gcm_bswap_epi64(%rip), %xmm9, %xmm8
 8194        vpaddd	L_avx1_aes_gcm_one(%rip), %xmm9, %xmm9
 8195        vmovdqu	%xmm9, 128(%rsp)
 8196        vpxor	(%r15), %xmm8, %xmm8
 8197        vpclmulqdq	$16, %xmm5, %xmm6, %xmm10
 8198        vaesenc	16(%r15), %xmm8, %xmm8
 8199        vaesenc	32(%r15), %xmm8, %xmm8
 8200        vpclmulqdq	$0x01, %xmm5, %xmm6, %xmm11
 8201        vaesenc	48(%r15), %xmm8, %xmm8
 8202        vaesenc	64(%r15), %xmm8, %xmm8
 8203        vpclmulqdq	$0x00, %xmm5, %xmm6, %xmm12
 8204        vaesenc	80(%r15), %xmm8, %xmm8
 8205        vpclmulqdq	$0x11, %xmm5, %xmm6, %xmm1
 8206        vaesenc	96(%r15), %xmm8, %xmm8
 8207        vpxor	%xmm11, %xmm10, %xmm10
 8208        vpslldq	$8, %xmm10, %xmm2
 8209        vpsrldq	$8, %xmm10, %xmm10
 8210        vaesenc	112(%r15), %xmm8, %xmm8
 8211        vpxor	%xmm12, %xmm2, %xmm2
 8212        vpxor	%xmm10, %xmm1, %xmm3
 8213        vmovdqa	L_avx1_aes_gcm_mod2_128(%rip), %xmm0
 8214        vpclmulqdq	$16, %xmm0, %xmm2, %xmm11
 8215        vaesenc	128(%r15), %xmm8, %xmm8
 8216        vpshufd	$0x4e, %xmm2, %xmm10
 8217        vpxor	%xmm11, %xmm10, %xmm10
 8218        vpclmulqdq	$16, %xmm0, %xmm10, %xmm11
 8219        vaesenc	144(%r15), %xmm8, %xmm8
 8220        vpshufd	$0x4e, %xmm10, %xmm10
 8221        vpxor	%xmm11, %xmm10, %xmm10
 8222        vpxor	%xmm3, %xmm10, %xmm6
 8223        cmpl	$11, %r10d
 8224        vmovdqa	160(%r15), %xmm9
 8225        jl	L_AES_GCM_encrypt_avx1_aesenc_gfmul_last
 8226        vaesenc	%xmm9, %xmm8, %xmm8
 8227        vaesenc	176(%r15), %xmm8, %xmm8
 8228        cmpl	$13, %r10d
 8229        vmovdqa	192(%r15), %xmm9
 8230        jl	L_AES_GCM_encrypt_avx1_aesenc_gfmul_last
 8231        vaesenc	%xmm9, %xmm8, %xmm8
 8232        vaesenc	208(%r15), %xmm8, %xmm8
 8233        vmovdqa	224(%r15), %xmm9
 8234L_AES_GCM_encrypt_avx1_aesenc_gfmul_last:
 8235        vaesenclast	%xmm9, %xmm8, %xmm8
 8236        vmovdqa	%xmm13, %xmm0
 8237        vpxor	%xmm0, %xmm8, %xmm8
 8238        vmovdqu	%xmm8, (%rsi,%rbx,1)
 8239        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm8, %xmm8
 8240        addl	$16, %ebx
 8241        vpxor	%xmm8, %xmm6, %xmm6
 8242        cmpl	%r13d, %ebx
 8243        jl	L_AES_GCM_encrypt_avx1_last_block_start
 8244L_AES_GCM_encrypt_avx1_last_block_ghash:
 8245        # ghash_gfmul_red_avx
 8246        vpshufd	$0x4e, %xmm5, %xmm9
 8247        vpshufd	$0x4e, %xmm6, %xmm10
 8248        vpclmulqdq	$0x11, %xmm5, %xmm6, %xmm11
 8249        vpclmulqdq	$0x00, %xmm5, %xmm6, %xmm8
 8250        vpxor	%xmm5, %xmm9, %xmm9
 8251        vpxor	%xmm6, %xmm10, %xmm10
 8252        vpclmulqdq	$0x00, %xmm10, %xmm9, %xmm9
 8253        vpxor	%xmm8, %xmm9, %xmm9
 8254        vpxor	%xmm11, %xmm9, %xmm9
 8255        vpslldq	$8, %xmm9, %xmm10
 8256        vpsrldq	$8, %xmm9, %xmm9
 8257        vpxor	%xmm10, %xmm8, %xmm8
 8258        vpxor	%xmm9, %xmm11, %xmm6
 8259        vpslld	$31, %xmm8, %xmm12
 8260        vpslld	$30, %xmm8, %xmm13
 8261        vpslld	$25, %xmm8, %xmm14
 8262        vpxor	%xmm13, %xmm12, %xmm12
 8263        vpxor	%xmm14, %xmm12, %xmm12
 8264        vpsrldq	$4, %xmm12, %xmm13
 8265        vpslldq	$12, %xmm12, %xmm12
 8266        vpxor	%xmm12, %xmm8, %xmm8
 8267        vpsrld	$0x01, %xmm8, %xmm14
 8268        vpsrld	$2, %xmm8, %xmm10
 8269        vpsrld	$7, %xmm8, %xmm9
 8270        vpxor	%xmm10, %xmm14, %xmm14
 8271        vpxor	%xmm9, %xmm14, %xmm14
 8272        vpxor	%xmm13, %xmm14, %xmm14
 8273        vpxor	%xmm8, %xmm14, %xmm14
 8274        vpxor	%xmm14, %xmm6, %xmm6
 8275L_AES_GCM_encrypt_avx1_last_block_done:
 8276        movl	%r9d, %ecx
 8277        movl	%ecx, %edx
 8278        andl	$15, %ecx
 8279        jz	L_AES_GCM_encrypt_avx1_aesenc_last15_enc_avx_done
 8280        vmovdqu	128(%rsp), %xmm4
 8281        vpshufb	L_avx1_aes_gcm_bswap_epi64(%rip), %xmm4, %xmm4
 8282        vpxor	(%r15), %xmm4, %xmm4
 8283        vaesenc	16(%r15), %xmm4, %xmm4
 8284        vaesenc	32(%r15), %xmm4, %xmm4
 8285        vaesenc	48(%r15), %xmm4, %xmm4
 8286        vaesenc	64(%r15), %xmm4, %xmm4
 8287        vaesenc	80(%r15), %xmm4, %xmm4
 8288        vaesenc	96(%r15), %xmm4, %xmm4
 8289        vaesenc	112(%r15), %xmm4, %xmm4
 8290        vaesenc	128(%r15), %xmm4, %xmm4
 8291        vaesenc	144(%r15), %xmm4, %xmm4
 8292        cmpl	$11, %r10d
 8293        vmovdqa	160(%r15), %xmm9
 8294        jl	L_AES_GCM_encrypt_avx1_aesenc_last15_enc_avx_aesenc_avx_last
 8295        vaesenc	%xmm9, %xmm4, %xmm4
 8296        vaesenc	176(%r15), %xmm4, %xmm4
 8297        cmpl	$13, %r10d
 8298        vmovdqa	192(%r15), %xmm9
 8299        jl	L_AES_GCM_encrypt_avx1_aesenc_last15_enc_avx_aesenc_avx_last
 8300        vaesenc	%xmm9, %xmm4, %xmm4
 8301        vaesenc	208(%r15), %xmm4, %xmm4
 8302        vmovdqa	224(%r15), %xmm9
 8303L_AES_GCM_encrypt_avx1_aesenc_last15_enc_avx_aesenc_avx_last:
 8304        vaesenclast	%xmm9, %xmm4, %xmm4
 8305        subq	$16, %rsp
 8306        xorl	%ecx, %ecx
 8307        vmovdqu	%xmm4, (%rsp)
 8308L_AES_GCM_encrypt_avx1_aesenc_last15_enc_avx_loop:
 8309        movzbl	(%rdi,%rbx,1), %r13d
 8310        xorb	(%rsp,%rcx,1), %r13b
 8311        movb	%r13b, (%rsi,%rbx,1)
 8312        movb	%r13b, (%rsp,%rcx,1)
 8313        incl	%ebx
 8314        incl	%ecx
 8315        cmpl	%edx, %ebx
 8316        jl	L_AES_GCM_encrypt_avx1_aesenc_last15_enc_avx_loop
 8317        xorq	%r13, %r13
 8318        cmpl	$16, %ecx
 8319        je	L_AES_GCM_encrypt_avx1_aesenc_last15_enc_avx_finish_enc
 8320L_AES_GCM_encrypt_avx1_aesenc_last15_enc_avx_byte_loop:
 8321        movb	%r13b, (%rsp,%rcx,1)
 8322        incl	%ecx
 8323        cmpl	$16, %ecx
 8324        jl	L_AES_GCM_encrypt_avx1_aesenc_last15_enc_avx_byte_loop
 8325L_AES_GCM_encrypt_avx1_aesenc_last15_enc_avx_finish_enc:
 8326        vmovdqu	(%rsp), %xmm4
 8327        addq	$16, %rsp
 8328        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm4, %xmm4
 8329        vpxor	%xmm4, %xmm6, %xmm6
 8330        # ghash_gfmul_red_avx
 8331        vpshufd	$0x4e, %xmm5, %xmm9
 8332        vpshufd	$0x4e, %xmm6, %xmm10
 8333        vpclmulqdq	$0x11, %xmm5, %xmm6, %xmm11
 8334        vpclmulqdq	$0x00, %xmm5, %xmm6, %xmm8
 8335        vpxor	%xmm5, %xmm9, %xmm9
 8336        vpxor	%xmm6, %xmm10, %xmm10
 8337        vpclmulqdq	$0x00, %xmm10, %xmm9, %xmm9
 8338        vpxor	%xmm8, %xmm9, %xmm9
 8339        vpxor	%xmm11, %xmm9, %xmm9
 8340        vpslldq	$8, %xmm9, %xmm10
 8341        vpsrldq	$8, %xmm9, %xmm9
 8342        vpxor	%xmm10, %xmm8, %xmm8
 8343        vpxor	%xmm9, %xmm11, %xmm6
 8344        vpslld	$31, %xmm8, %xmm12
 8345        vpslld	$30, %xmm8, %xmm13
 8346        vpslld	$25, %xmm8, %xmm14
 8347        vpxor	%xmm13, %xmm12, %xmm12
 8348        vpxor	%xmm14, %xmm12, %xmm12
 8349        vpsrldq	$4, %xmm12, %xmm13
 8350        vpslldq	$12, %xmm12, %xmm12
 8351        vpxor	%xmm12, %xmm8, %xmm8
 8352        vpsrld	$0x01, %xmm8, %xmm14
 8353        vpsrld	$2, %xmm8, %xmm10
 8354        vpsrld	$7, %xmm8, %xmm9
 8355        vpxor	%xmm10, %xmm14, %xmm14
 8356        vpxor	%xmm9, %xmm14, %xmm14
 8357        vpxor	%xmm13, %xmm14, %xmm14
 8358        vpxor	%xmm8, %xmm14, %xmm14
 8359        vpxor	%xmm14, %xmm6, %xmm6
 8360L_AES_GCM_encrypt_avx1_aesenc_last15_enc_avx_done:
 8361L_AES_GCM_encrypt_avx1_done_enc:
 8362        movl	%r9d, %edx
 8363        movl	%r11d, %ecx
 8364        shlq	$3, %rdx
 8365        shlq	$3, %rcx
 8366        vmovq	%rdx, %xmm0
 8367        vmovq	%rcx, %xmm1
 8368        vpunpcklqdq	%xmm1, %xmm0, %xmm0
 8369        vpxor	%xmm0, %xmm6, %xmm6
 8370        # ghash_gfmul_red_avx
 8371        vpshufd	$0x4e, %xmm5, %xmm9
 8372        vpshufd	$0x4e, %xmm6, %xmm10
 8373        vpclmulqdq	$0x11, %xmm5, %xmm6, %xmm11
 8374        vpclmulqdq	$0x00, %xmm5, %xmm6, %xmm8
 8375        vpxor	%xmm5, %xmm9, %xmm9
 8376        vpxor	%xmm6, %xmm10, %xmm10
 8377        vpclmulqdq	$0x00, %xmm10, %xmm9, %xmm9
 8378        vpxor	%xmm8, %xmm9, %xmm9
 8379        vpxor	%xmm11, %xmm9, %xmm9
 8380        vpslldq	$8, %xmm9, %xmm10
 8381        vpsrldq	$8, %xmm9, %xmm9
 8382        vpxor	%xmm10, %xmm8, %xmm8
 8383        vpxor	%xmm9, %xmm11, %xmm6
 8384        vpslld	$31, %xmm8, %xmm12
 8385        vpslld	$30, %xmm8, %xmm13
 8386        vpslld	$25, %xmm8, %xmm14
 8387        vpxor	%xmm13, %xmm12, %xmm12
 8388        vpxor	%xmm14, %xmm12, %xmm12
 8389        vpsrldq	$4, %xmm12, %xmm13
 8390        vpslldq	$12, %xmm12, %xmm12
 8391        vpxor	%xmm12, %xmm8, %xmm8
 8392        vpsrld	$0x01, %xmm8, %xmm14
 8393        vpsrld	$2, %xmm8, %xmm10
 8394        vpsrld	$7, %xmm8, %xmm9
 8395        vpxor	%xmm10, %xmm14, %xmm14
 8396        vpxor	%xmm9, %xmm14, %xmm14
 8397        vpxor	%xmm13, %xmm14, %xmm14
 8398        vpxor	%xmm8, %xmm14, %xmm14
 8399        vpxor	%xmm14, %xmm6, %xmm6
 8400        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm6, %xmm6
 8401        vmovdqu	144(%rsp), %xmm0
 8402        vpxor	%xmm6, %xmm0, %xmm0
 8403        cmpl	$16, %r14d
 8404        je	L_AES_GCM_encrypt_avx1_store_tag_16
 8405        xorq	%rcx, %rcx
 8406        vmovdqu	%xmm0, (%rsp)
 8407L_AES_GCM_encrypt_avx1_store_tag_loop:
 8408        movzbl	(%rsp,%rcx,1), %r13d
 8409        movb	%r13b, (%r8,%rcx,1)
 8410        incl	%ecx
 8411        cmpl	%r14d, %ecx
 8412        jne	L_AES_GCM_encrypt_avx1_store_tag_loop
 8413        jmp	L_AES_GCM_encrypt_avx1_store_tag_done
 8414L_AES_GCM_encrypt_avx1_store_tag_16:
 8415        vmovdqu	%xmm0, (%r8)
 8416L_AES_GCM_encrypt_avx1_store_tag_done:
 8417        vzeroupper
 8418        addq	$0xa0, %rsp
 8419        popq	%r15
 8420        popq	%r14
 8421        popq	%rbx
 8422        popq	%r12
 8423        popq	%r13
 8424        repz retq
 8425#ifndef __APPLE__
 8426.size	AES_GCM_encrypt_avx1,.-AES_GCM_encrypt_avx1
 8427#endif /* __APPLE__ */
 8428#ifndef __APPLE__
 8429.text
 8430.globl	AES_GCM_decrypt_avx1
 8431.type	AES_GCM_decrypt_avx1,@function
 8432.align	16
 8433AES_GCM_decrypt_avx1:
 8434#else
 8435.section	__TEXT,__text
 8436.globl	_AES_GCM_decrypt_avx1
 8437.p2align	4
 8438_AES_GCM_decrypt_avx1:
 8439#endif /* __APPLE__ */
 8440        pushq	%r13
 8441        pushq	%r12
 8442        pushq	%rbx
 8443        pushq	%r14
 8444        pushq	%r15
 8445        pushq	%rbp
 8446        movq	%rdx, %r12
 8447        movq	%rcx, %rax
 8448        movl	56(%rsp), %r11d
 8449        movl	64(%rsp), %ebx
 8450        movl	72(%rsp), %r14d
 8451        movq	80(%rsp), %r15
 8452        movl	88(%rsp), %r10d
 8453        movq	96(%rsp), %rbp
 8454        subq	$0xa8, %rsp
 8455        vpxor	%xmm4, %xmm4, %xmm4
 8456        vpxor	%xmm6, %xmm6, %xmm6
 8457        cmpl	$12, %ebx
 8458        movl	%ebx, %edx
 8459        jne	L_AES_GCM_decrypt_avx1_iv_not_12
 8460        # # Calculate values when IV is 12 bytes
 8461        # Set counter based on IV
 8462        movl	$0x1000000, %ecx
 8463        vmovq	(%rax), %xmm4
 8464        vpinsrd	$2, 8(%rax), %xmm4, %xmm4
 8465        vpinsrd	$3, %ecx, %xmm4, %xmm4
 8466        # H = Encrypt X(=0) and T = Encrypt counter
 8467        vmovdqa	(%r15), %xmm5
 8468        vpxor	%xmm5, %xmm4, %xmm1
 8469        vmovdqa	16(%r15), %xmm7
 8470        vaesenc	%xmm7, %xmm5, %xmm5
 8471        vaesenc	%xmm7, %xmm1, %xmm1
 8472        vmovdqa	32(%r15), %xmm7
 8473        vaesenc	%xmm7, %xmm5, %xmm5
 8474        vaesenc	%xmm7, %xmm1, %xmm1
 8475        vmovdqa	48(%r15), %xmm7
 8476        vaesenc	%xmm7, %xmm5, %xmm5
 8477        vaesenc	%xmm7, %xmm1, %xmm1
 8478        vmovdqa	64(%r15), %xmm7
 8479        vaesenc	%xmm7, %xmm5, %xmm5
 8480        vaesenc	%xmm7, %xmm1, %xmm1
 8481        vmovdqa	80(%r15), %xmm7
 8482        vaesenc	%xmm7, %xmm5, %xmm5
 8483        vaesenc	%xmm7, %xmm1, %xmm1
 8484        vmovdqa	96(%r15), %xmm7
 8485        vaesenc	%xmm7, %xmm5, %xmm5
 8486        vaesenc	%xmm7, %xmm1, %xmm1
 8487        vmovdqa	112(%r15), %xmm7
 8488        vaesenc	%xmm7, %xmm5, %xmm5
 8489        vaesenc	%xmm7, %xmm1, %xmm1
 8490        vmovdqa	128(%r15), %xmm7
 8491        vaesenc	%xmm7, %xmm5, %xmm5
 8492        vaesenc	%xmm7, %xmm1, %xmm1
 8493        vmovdqa	144(%r15), %xmm7
 8494        vaesenc	%xmm7, %xmm5, %xmm5
 8495        vaesenc	%xmm7, %xmm1, %xmm1
 8496        cmpl	$11, %r10d
 8497        vmovdqa	160(%r15), %xmm7
 8498        jl	L_AES_GCM_decrypt_avx1_calc_iv_12_last
 8499        vaesenc	%xmm7, %xmm5, %xmm5
 8500        vaesenc	%xmm7, %xmm1, %xmm1
 8501        vmovdqa	176(%r15), %xmm7
 8502        vaesenc	%xmm7, %xmm5, %xmm5
 8503        vaesenc	%xmm7, %xmm1, %xmm1
 8504        cmpl	$13, %r10d
 8505        vmovdqa	192(%r15), %xmm7
 8506        jl	L_AES_GCM_decrypt_avx1_calc_iv_12_last
 8507        vaesenc	%xmm7, %xmm5, %xmm5
 8508        vaesenc	%xmm7, %xmm1, %xmm1
 8509        vmovdqa	208(%r15), %xmm7
 8510        vaesenc	%xmm7, %xmm5, %xmm5
 8511        vaesenc	%xmm7, %xmm1, %xmm1
 8512        vmovdqa	224(%r15), %xmm7
 8513L_AES_GCM_decrypt_avx1_calc_iv_12_last:
 8514        vaesenclast	%xmm7, %xmm5, %xmm5
 8515        vaesenclast	%xmm7, %xmm1, %xmm1
 8516        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm5, %xmm5
 8517        vmovdqu	%xmm1, 144(%rsp)
 8518        jmp	L_AES_GCM_decrypt_avx1_iv_done
 8519L_AES_GCM_decrypt_avx1_iv_not_12:
 8520        # Calculate values when IV is not 12 bytes
 8521        # H = Encrypt X(=0)
 8522        vmovdqa	(%r15), %xmm5
 8523        vaesenc	16(%r15), %xmm5, %xmm5
 8524        vaesenc	32(%r15), %xmm5, %xmm5
 8525        vaesenc	48(%r15), %xmm5, %xmm5
 8526        vaesenc	64(%r15), %xmm5, %xmm5
 8527        vaesenc	80(%r15), %xmm5, %xmm5
 8528        vaesenc	96(%r15), %xmm5, %xmm5
 8529        vaesenc	112(%r15), %xmm5, %xmm5
 8530        vaesenc	128(%r15), %xmm5, %xmm5
 8531        vaesenc	144(%r15), %xmm5, %xmm5
 8532        cmpl	$11, %r10d
 8533        vmovdqa	160(%r15), %xmm9
 8534        jl	L_AES_GCM_decrypt_avx1_calc_iv_1_aesenc_avx_last
 8535        vaesenc	%xmm9, %xmm5, %xmm5
 8536        vaesenc	176(%r15), %xmm5, %xmm5
 8537        cmpl	$13, %r10d
 8538        vmovdqa	192(%r15), %xmm9
 8539        jl	L_AES_GCM_decrypt_avx1_calc_iv_1_aesenc_avx_last
 8540        vaesenc	%xmm9, %xmm5, %xmm5
 8541        vaesenc	208(%r15), %xmm5, %xmm5
 8542        vmovdqa	224(%r15), %xmm9
 8543L_AES_GCM_decrypt_avx1_calc_iv_1_aesenc_avx_last:
 8544        vaesenclast	%xmm9, %xmm5, %xmm5
 8545        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm5, %xmm5
 8546        # Calc counter
 8547        # Initialization vector
 8548        cmpl	$0x00, %edx
 8549        movq	$0x00, %rcx
 8550        je	L_AES_GCM_decrypt_avx1_calc_iv_done
 8551        cmpl	$16, %edx
 8552        jl	L_AES_GCM_decrypt_avx1_calc_iv_lt16
 8553        andl	$0xfffffff0, %edx
 8554L_AES_GCM_decrypt_avx1_calc_iv_16_loop:
 8555        vmovdqu	(%rax,%rcx,1), %xmm8
 8556        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm8, %xmm8
 8557        vpxor	%xmm8, %xmm4, %xmm4
 8558        # ghash_gfmul_avx
 8559        vpshufd	$0x4e, %xmm4, %xmm1
 8560        vpshufd	$0x4e, %xmm5, %xmm2
 8561        vpclmulqdq	$0x11, %xmm4, %xmm5, %xmm3
 8562        vpclmulqdq	$0x00, %xmm4, %xmm5, %xmm0
 8563        vpxor	%xmm4, %xmm1, %xmm1
 8564        vpxor	%xmm5, %xmm2, %xmm2
 8565        vpclmulqdq	$0x00, %xmm2, %xmm1, %xmm1
 8566        vpxor	%xmm0, %xmm1, %xmm1
 8567        vpxor	%xmm3, %xmm1, %xmm1
 8568        vmovdqa	%xmm0, %xmm7
 8569        vmovdqa	%xmm3, %xmm4
 8570        vpslldq	$8, %xmm1, %xmm2
 8571        vpsrldq	$8, %xmm1, %xmm1
 8572        vpxor	%xmm2, %xmm7, %xmm7
 8573        vpxor	%xmm1, %xmm4, %xmm4
 8574        vpsrld	$31, %xmm7, %xmm0
 8575        vpsrld	$31, %xmm4, %xmm1
 8576        vpslld	$0x01, %xmm7, %xmm7
 8577        vpslld	$0x01, %xmm4, %xmm4
 8578        vpsrldq	$12, %xmm0, %xmm2
 8579        vpslldq	$4, %xmm0, %xmm0
 8580        vpslldq	$4, %xmm1, %xmm1
 8581        vpor	%xmm2, %xmm4, %xmm4
 8582        vpor	%xmm0, %xmm7, %xmm7
 8583        vpor	%xmm1, %xmm4, %xmm4
 8584        vpslld	$31, %xmm7, %xmm0
 8585        vpslld	$30, %xmm7, %xmm1
 8586        vpslld	$25, %xmm7, %xmm2
 8587        vpxor	%xmm1, %xmm0, %xmm0
 8588        vpxor	%xmm2, %xmm0, %xmm0
 8589        vmovdqa	%xmm0, %xmm1
 8590        vpsrldq	$4, %xmm1, %xmm1
 8591        vpslldq	$12, %xmm0, %xmm0
 8592        vpxor	%xmm0, %xmm7, %xmm7
 8593        vpsrld	$0x01, %xmm7, %xmm2
 8594        vpsrld	$2, %xmm7, %xmm3
 8595        vpsrld	$7, %xmm7, %xmm0
 8596        vpxor	%xmm3, %xmm2, %xmm2
 8597        vpxor	%xmm0, %xmm2, %xmm2
 8598        vpxor	%xmm1, %xmm2, %xmm2
 8599        vpxor	%xmm7, %xmm2, %xmm2
 8600        vpxor	%xmm2, %xmm4, %xmm4
 8601        addl	$16, %ecx
 8602        cmpl	%edx, %ecx
 8603        jl	L_AES_GCM_decrypt_avx1_calc_iv_16_loop
 8604        movl	%ebx, %edx
 8605        cmpl	%edx, %ecx
 8606        je	L_AES_GCM_decrypt_avx1_calc_iv_done
 8607L_AES_GCM_decrypt_avx1_calc_iv_lt16:
 8608        subq	$16, %rsp
 8609        vpxor	%xmm8, %xmm8, %xmm8
 8610        xorl	%ebx, %ebx
 8611        vmovdqu	%xmm8, (%rsp)
 8612L_AES_GCM_decrypt_avx1_calc_iv_loop:
 8613        movzbl	(%rax,%rcx,1), %r13d
 8614        movb	%r13b, (%rsp,%rbx,1)
 8615        incl	%ecx
 8616        incl	%ebx
 8617        cmpl	%edx, %ecx
 8618        jl	L_AES_GCM_decrypt_avx1_calc_iv_loop
 8619        vmovdqu	(%rsp), %xmm8
 8620        addq	$16, %rsp
 8621        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm8, %xmm8
 8622        vpxor	%xmm8, %xmm4, %xmm4
 8623        # ghash_gfmul_avx
 8624        vpshufd	$0x4e, %xmm4, %xmm1
 8625        vpshufd	$0x4e, %xmm5, %xmm2
 8626        vpclmulqdq	$0x11, %xmm4, %xmm5, %xmm3
 8627        vpclmulqdq	$0x00, %xmm4, %xmm5, %xmm0
 8628        vpxor	%xmm4, %xmm1, %xmm1
 8629        vpxor	%xmm5, %xmm2, %xmm2
 8630        vpclmulqdq	$0x00, %xmm2, %xmm1, %xmm1
 8631        vpxor	%xmm0, %xmm1, %xmm1
 8632        vpxor	%xmm3, %xmm1, %xmm1
 8633        vmovdqa	%xmm0, %xmm7
 8634        vmovdqa	%xmm3, %xmm4
 8635        vpslldq	$8, %xmm1, %xmm2
 8636        vpsrldq	$8, %xmm1, %xmm1
 8637        vpxor	%xmm2, %xmm7, %xmm7
 8638        vpxor	%xmm1, %xmm4, %xmm4
 8639        vpsrld	$31, %xmm7, %xmm0
 8640        vpsrld	$31, %xmm4, %xmm1
 8641        vpslld	$0x01, %xmm7, %xmm7
 8642        vpslld	$0x01, %xmm4, %xmm4
 8643        vpsrldq	$12, %xmm0, %xmm2
 8644        vpslldq	$4, %xmm0, %xmm0
 8645        vpslldq	$4, %xmm1, %xmm1
 8646        vpor	%xmm2, %xmm4, %xmm4
 8647        vpor	%xmm0, %xmm7, %xmm7
 8648        vpor	%xmm1, %xmm4, %xmm4
 8649        vpslld	$31, %xmm7, %xmm0
 8650        vpslld	$30, %xmm7, %xmm1
 8651        vpslld	$25, %xmm7, %xmm2
 8652        vpxor	%xmm1, %xmm0, %xmm0
 8653        vpxor	%xmm2, %xmm0, %xmm0
 8654        vmovdqa	%xmm0, %xmm1
 8655        vpsrldq	$4, %xmm1, %xmm1
 8656        vpslldq	$12, %xmm0, %xmm0
 8657        vpxor	%xmm0, %xmm7, %xmm7
 8658        vpsrld	$0x01, %xmm7, %xmm2
 8659        vpsrld	$2, %xmm7, %xmm3
 8660        vpsrld	$7, %xmm7, %xmm0
 8661        vpxor	%xmm3, %xmm2, %xmm2
 8662        vpxor	%xmm0, %xmm2, %xmm2
 8663        vpxor	%xmm1, %xmm2, %xmm2
 8664        vpxor	%xmm7, %xmm2, %xmm2
 8665        vpxor	%xmm2, %xmm4, %xmm4
 8666L_AES_GCM_decrypt_avx1_calc_iv_done:
 8667        # T = Encrypt counter
 8668        vpxor	%xmm0, %xmm0, %xmm0
 8669        shll	$3, %edx
 8670        vmovq	%rdx, %xmm0
 8671        vpxor	%xmm0, %xmm4, %xmm4
 8672        # ghash_gfmul_avx
 8673        vpshufd	$0x4e, %xmm4, %xmm1
 8674        vpshufd	$0x4e, %xmm5, %xmm2
 8675        vpclmulqdq	$0x11, %xmm4, %xmm5, %xmm3
 8676        vpclmulqdq	$0x00, %xmm4, %xmm5, %xmm0
 8677        vpxor	%xmm4, %xmm1, %xmm1
 8678        vpxor	%xmm5, %xmm2, %xmm2
 8679        vpclmulqdq	$0x00, %xmm2, %xmm1, %xmm1
 8680        vpxor	%xmm0, %xmm1, %xmm1
 8681        vpxor	%xmm3, %xmm1, %xmm1
 8682        vmovdqa	%xmm0, %xmm7
 8683        vmovdqa	%xmm3, %xmm4
 8684        vpslldq	$8, %xmm1, %xmm2
 8685        vpsrldq	$8, %xmm1, %xmm1
 8686        vpxor	%xmm2, %xmm7, %xmm7
 8687        vpxor	%xmm1, %xmm4, %xmm4
 8688        vpsrld	$31, %xmm7, %xmm0
 8689        vpsrld	$31, %xmm4, %xmm1
 8690        vpslld	$0x01, %xmm7, %xmm7
 8691        vpslld	$0x01, %xmm4, %xmm4
 8692        vpsrldq	$12, %xmm0, %xmm2
 8693        vpslldq	$4, %xmm0, %xmm0
 8694        vpslldq	$4, %xmm1, %xmm1
 8695        vpor	%xmm2, %xmm4, %xmm4
 8696        vpor	%xmm0, %xmm7, %xmm7
 8697        vpor	%xmm1, %xmm4, %xmm4
 8698        vpslld	$31, %xmm7, %xmm0
 8699        vpslld	$30, %xmm7, %xmm1
 8700        vpslld	$25, %xmm7, %xmm2
 8701        vpxor	%xmm1, %xmm0, %xmm0
 8702        vpxor	%xmm2, %xmm0, %xmm0
 8703        vmovdqa	%xmm0, %xmm1
 8704        vpsrldq	$4, %xmm1, %xmm1
 8705        vpslldq	$12, %xmm0, %xmm0
 8706        vpxor	%xmm0, %xmm7, %xmm7
 8707        vpsrld	$0x01, %xmm7, %xmm2
 8708        vpsrld	$2, %xmm7, %xmm3
 8709        vpsrld	$7, %xmm7, %xmm0
 8710        vpxor	%xmm3, %xmm2, %xmm2
 8711        vpxor	%xmm0, %xmm2, %xmm2
 8712        vpxor	%xmm1, %xmm2, %xmm2
 8713        vpxor	%xmm7, %xmm2, %xmm2
 8714        vpxor	%xmm2, %xmm4, %xmm4
 8715        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm4, %xmm4
 8716        #   Encrypt counter
 8717        vmovdqa	(%r15), %xmm8
 8718        vpxor	%xmm4, %xmm8, %xmm8
 8719        vaesenc	16(%r15), %xmm8, %xmm8
 8720        vaesenc	32(%r15), %xmm8, %xmm8
 8721        vaesenc	48(%r15), %xmm8, %xmm8
 8722        vaesenc	64(%r15), %xmm8, %xmm8
 8723        vaesenc	80(%r15), %xmm8, %xmm8
 8724        vaesenc	96(%r15), %xmm8, %xmm8
 8725        vaesenc	112(%r15), %xmm8, %xmm8
 8726        vaesenc	128(%r15), %xmm8, %xmm8
 8727        vaesenc	144(%r15), %xmm8, %xmm8
 8728        cmpl	$11, %r10d
 8729        vmovdqa	160(%r15), %xmm9
 8730        jl	L_AES_GCM_decrypt_avx1_calc_iv_2_aesenc_avx_last
 8731        vaesenc	%xmm9, %xmm8, %xmm8
 8732        vaesenc	176(%r15), %xmm8, %xmm8
 8733        cmpl	$13, %r10d
 8734        vmovdqa	192(%r15), %xmm9
 8735        jl	L_AES_GCM_decrypt_avx1_calc_iv_2_aesenc_avx_last
 8736        vaesenc	%xmm9, %xmm8, %xmm8
 8737        vaesenc	208(%r15), %xmm8, %xmm8
 8738        vmovdqa	224(%r15), %xmm9
 8739L_AES_GCM_decrypt_avx1_calc_iv_2_aesenc_avx_last:
 8740        vaesenclast	%xmm9, %xmm8, %xmm8
 8741        vmovdqu	%xmm8, 144(%rsp)
 8742L_AES_GCM_decrypt_avx1_iv_done:
 8743        # Additional authentication data
 8744        movl	%r11d, %edx
 8745        cmpl	$0x00, %edx
 8746        je	L_AES_GCM_decrypt_avx1_calc_aad_done
 8747        xorl	%ecx, %ecx
 8748        cmpl	$16, %edx
 8749        jl	L_AES_GCM_decrypt_avx1_calc_aad_lt16
 8750        andl	$0xfffffff0, %edx
 8751L_AES_GCM_decrypt_avx1_calc_aad_16_loop:
 8752        vmovdqu	(%r12,%rcx,1), %xmm8
 8753        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm8, %xmm8
 8754        vpxor	%xmm8, %xmm6, %xmm6
 8755        # ghash_gfmul_avx
 8756        vpshufd	$0x4e, %xmm6, %xmm1
 8757        vpshufd	$0x4e, %xmm5, %xmm2
 8758        vpclmulqdq	$0x11, %xmm6, %xmm5, %xmm3
 8759        vpclmulqdq	$0x00, %xmm6, %xmm5, %xmm0
 8760        vpxor	%xmm6, %xmm1, %xmm1
 8761        vpxor	%xmm5, %xmm2, %xmm2
 8762        vpclmulqdq	$0x00, %xmm2, %xmm1, %xmm1
 8763        vpxor	%xmm0, %xmm1, %xmm1
 8764        vpxor	%xmm3, %xmm1, %xmm1
 8765        vmovdqa	%xmm0, %xmm7
 8766        vmovdqa	%xmm3, %xmm6
 8767        vpslldq	$8, %xmm1, %xmm2
 8768        vpsrldq	$8, %xmm1, %xmm1
 8769        vpxor	%xmm2, %xmm7, %xmm7
 8770        vpxor	%xmm1, %xmm6, %xmm6
 8771        vpsrld	$31, %xmm7, %xmm0
 8772        vpsrld	$31, %xmm6, %xmm1
 8773        vpslld	$0x01, %xmm7, %xmm7
 8774        vpslld	$0x01, %xmm6, %xmm6
 8775        vpsrldq	$12, %xmm0, %xmm2
 8776        vpslldq	$4, %xmm0, %xmm0
 8777        vpslldq	$4, %xmm1, %xmm1
 8778        vpor	%xmm2, %xmm6, %xmm6
 8779        vpor	%xmm0, %xmm7, %xmm7
 8780        vpor	%xmm1, %xmm6, %xmm6
 8781        vpslld	$31, %xmm7, %xmm0
 8782        vpslld	$30, %xmm7, %xmm1
 8783        vpslld	$25, %xmm7, %xmm2
 8784        vpxor	%xmm1, %xmm0, %xmm0
 8785        vpxor	%xmm2, %xmm0, %xmm0
 8786        vmovdqa	%xmm0, %xmm1
 8787        vpsrldq	$4, %xmm1, %xmm1
 8788        vpslldq	$12, %xmm0, %xmm0
 8789        vpxor	%xmm0, %xmm7, %xmm7
 8790        vpsrld	$0x01, %xmm7, %xmm2
 8791        vpsrld	$2, %xmm7, %xmm3
 8792        vpsrld	$7, %xmm7, %xmm0
 8793        vpxor	%xmm3, %xmm2, %xmm2
 8794        vpxor	%xmm0, %xmm2, %xmm2
 8795        vpxor	%xmm1, %xmm2, %xmm2
 8796        vpxor	%xmm7, %xmm2, %xmm2
 8797        vpxor	%xmm2, %xmm6, %xmm6
 8798        addl	$16, %ecx
 8799        cmpl	%edx, %ecx
 8800        jl	L_AES_GCM_decrypt_avx1_calc_aad_16_loop
 8801        movl	%r11d, %edx
 8802        cmpl	%edx, %ecx
 8803        je	L_AES_GCM_decrypt_avx1_calc_aad_done
 8804L_AES_GCM_decrypt_avx1_calc_aad_lt16:
 8805        subq	$16, %rsp
 8806        vpxor	%xmm8, %xmm8, %xmm8
 8807        xorl	%ebx, %ebx
 8808        vmovdqu	%xmm8, (%rsp)
 8809L_AES_GCM_decrypt_avx1_calc_aad_loop:
 8810        movzbl	(%r12,%rcx,1), %r13d
 8811        movb	%r13b, (%rsp,%rbx,1)
 8812        incl	%ecx
 8813        incl	%ebx
 8814        cmpl	%edx, %ecx
 8815        jl	L_AES_GCM_decrypt_avx1_calc_aad_loop
 8816        vmovdqu	(%rsp), %xmm8
 8817        addq	$16, %rsp
 8818        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm8, %xmm8
 8819        vpxor	%xmm8, %xmm6, %xmm6
 8820        # ghash_gfmul_avx
 8821        vpshufd	$0x4e, %xmm6, %xmm1
 8822        vpshufd	$0x4e, %xmm5, %xmm2
 8823        vpclmulqdq	$0x11, %xmm6, %xmm5, %xmm3
 8824        vpclmulqdq	$0x00, %xmm6, %xmm5, %xmm0
 8825        vpxor	%xmm6, %xmm1, %xmm1
 8826        vpxor	%xmm5, %xmm2, %xmm2
 8827        vpclmulqdq	$0x00, %xmm2, %xmm1, %xmm1
 8828        vpxor	%xmm0, %xmm1, %xmm1
 8829        vpxor	%xmm3, %xmm1, %xmm1
 8830        vmovdqa	%xmm0, %xmm7
 8831        vmovdqa	%xmm3, %xmm6
 8832        vpslldq	$8, %xmm1, %xmm2
 8833        vpsrldq	$8, %xmm1, %xmm1
 8834        vpxor	%xmm2, %xmm7, %xmm7
 8835        vpxor	%xmm1, %xmm6, %xmm6
 8836        vpsrld	$31, %xmm7, %xmm0
 8837        vpsrld	$31, %xmm6, %xmm1
 8838        vpslld	$0x01, %xmm7, %xmm7
 8839        vpslld	$0x01, %xmm6, %xmm6
 8840        vpsrldq	$12, %xmm0, %xmm2
 8841        vpslldq	$4, %xmm0, %xmm0
 8842        vpslldq	$4, %xmm1, %xmm1
 8843        vpor	%xmm2, %xmm6, %xmm6
 8844        vpor	%xmm0, %xmm7, %xmm7
 8845        vpor	%xmm1, %xmm6, %xmm6
 8846        vpslld	$31, %xmm7, %xmm0
 8847        vpslld	$30, %xmm7, %xmm1
 8848        vpslld	$25, %xmm7, %xmm2
 8849        vpxor	%xmm1, %xmm0, %xmm0
 8850        vpxor	%xmm2, %xmm0, %xmm0
 8851        vmovdqa	%xmm0, %xmm1
 8852        vpsrldq	$4, %xmm1, %xmm1
 8853        vpslldq	$12, %xmm0, %xmm0
 8854        vpxor	%xmm0, %xmm7, %xmm7
 8855        vpsrld	$0x01, %xmm7, %xmm2
 8856        vpsrld	$2, %xmm7, %xmm3
 8857        vpsrld	$7, %xmm7, %xmm0
 8858        vpxor	%xmm3, %xmm2, %xmm2
 8859        vpxor	%xmm0, %xmm2, %xmm2
 8860        vpxor	%xmm1, %xmm2, %xmm2
 8861        vpxor	%xmm7, %xmm2, %xmm2
 8862        vpxor	%xmm2, %xmm6, %xmm6
 8863L_AES_GCM_decrypt_avx1_calc_aad_done:
 8864        # Calculate counter and H
 8865        vpsrlq	$63, %xmm5, %xmm9
 8866        vpsllq	$0x01, %xmm5, %xmm8
 8867        vpslldq	$8, %xmm9, %xmm9
 8868        vpor	%xmm9, %xmm8, %xmm8
 8869        vpshufd	$0xff, %xmm5, %xmm5
 8870        vpsrad	$31, %xmm5, %xmm5
 8871        vpshufb	L_avx1_aes_gcm_bswap_epi64(%rip), %xmm4, %xmm4
 8872        vpand	L_avx1_aes_gcm_mod2_128(%rip), %xmm5, %xmm5
 8873        vpaddd	L_avx1_aes_gcm_one(%rip), %xmm4, %xmm4
 8874        vpxor	%xmm8, %xmm5, %xmm5
 8875        vmovdqu	%xmm4, 128(%rsp)
 8876        xorl	%ebx, %ebx
 8877        cmpl	$0x80, %r9d
 8878        movl	%r9d, %r13d
 8879        jl	L_AES_GCM_decrypt_avx1_done_128
 8880        andl	$0xffffff80, %r13d
 8881        vmovdqa	%xmm6, %xmm2
 8882        # H ^ 1
 8883        vmovdqu	%xmm5, (%rsp)
 8884        # H ^ 2
 8885        vpclmulqdq	$0x00, %xmm5, %xmm5, %xmm8
 8886        vpclmulqdq	$0x11, %xmm5, %xmm5, %xmm0
 8887        vpslld	$31, %xmm8, %xmm12
 8888        vpslld	$30, %xmm8, %xmm13
 8889        vpslld	$25, %xmm8, %xmm14
 8890        vpxor	%xmm13, %xmm12, %xmm12
 8891        vpxor	%xmm14, %xmm12, %xmm12
 8892        vpsrldq	$4, %xmm12, %xmm13
 8893        vpslldq	$12, %xmm12, %xmm12
 8894        vpxor	%xmm12, %xmm8, %xmm8
 8895        vpsrld	$0x01, %xmm8, %xmm14
 8896        vpsrld	$2, %xmm8, %xmm10
 8897        vpsrld	$7, %xmm8, %xmm9
 8898        vpxor	%xmm10, %xmm14, %xmm14
 8899        vpxor	%xmm9, %xmm14, %xmm14
 8900        vpxor	%xmm13, %xmm14, %xmm14
 8901        vpxor	%xmm8, %xmm14, %xmm14
 8902        vpxor	%xmm14, %xmm0, %xmm0
 8903        vmovdqu	%xmm0, 16(%rsp)
 8904        # H ^ 3
 8905        # ghash_gfmul_red_avx
 8906        vpshufd	$0x4e, %xmm5, %xmm9
 8907        vpshufd	$0x4e, %xmm0, %xmm10
 8908        vpclmulqdq	$0x11, %xmm5, %xmm0, %xmm11
 8909        vpclmulqdq	$0x00, %xmm5, %xmm0, %xmm8
 8910        vpxor	%xmm5, %xmm9, %xmm9
 8911        vpxor	%xmm0, %xmm10, %xmm10
 8912        vpclmulqdq	$0x00, %xmm10, %xmm9, %xmm9
 8913        vpxor	%xmm8, %xmm9, %xmm9
 8914        vpxor	%xmm11, %xmm9, %xmm9
 8915        vpslldq	$8, %xmm9, %xmm10
 8916        vpsrldq	$8, %xmm9, %xmm9
 8917        vpxor	%xmm10, %xmm8, %xmm8
 8918        vpxor	%xmm9, %xmm11, %xmm1
 8919        vpslld	$31, %xmm8, %xmm12
 8920        vpslld	$30, %xmm8, %xmm13
 8921        vpslld	$25, %xmm8, %xmm14
 8922        vpxor	%xmm13, %xmm12, %xmm12
 8923        vpxor	%xmm14, %xmm12, %xmm12
 8924        vpsrldq	$4, %xmm12, %xmm13
 8925        vpslldq	$12, %xmm12, %xmm12
 8926        vpxor	%xmm12, %xmm8, %xmm8
 8927        vpsrld	$0x01, %xmm8, %xmm14
 8928        vpsrld	$2, %xmm8, %xmm10
 8929        vpsrld	$7, %xmm8, %xmm9
 8930        vpxor	%xmm10, %xmm14, %xmm14
 8931        vpxor	%xmm9, %xmm14, %xmm14
 8932        vpxor	%xmm13, %xmm14, %xmm14
 8933        vpxor	%xmm8, %xmm14, %xmm14
 8934        vpxor	%xmm14, %xmm1, %xmm1
 8935        vmovdqu	%xmm1, 32(%rsp)
 8936        # H ^ 4
 8937        vpclmulqdq	$0x00, %xmm0, %xmm0, %xmm8
 8938        vpclmulqdq	$0x11, %xmm0, %xmm0, %xmm3
 8939        vpslld	$31, %xmm8, %xmm12
 8940        vpslld	$30, %xmm8, %xmm13
 8941        vpslld	$25, %xmm8, %xmm14
 8942        vpxor	%xmm13, %xmm12, %xmm12
 8943        vpxor	%xmm14, %xmm12, %xmm12
 8944        vpsrldq	$4, %xmm12, %xmm13
 8945        vpslldq	$12, %xmm12, %xmm12
 8946        vpxor	%xmm12, %xmm8, %xmm8
 8947        vpsrld	$0x01, %xmm8, %xmm14
 8948        vpsrld	$2, %xmm8, %xmm10
 8949        vpsrld	$7, %xmm8, %xmm9
 8950        vpxor	%xmm10, %xmm14, %xmm14
 8951        vpxor	%xmm9, %xmm14, %xmm14
 8952        vpxor	%xmm13, %xmm14, %xmm14
 8953        vpxor	%xmm8, %xmm14, %xmm14
 8954        vpxor	%xmm14, %xmm3, %xmm3
 8955        vmovdqu	%xmm3, 48(%rsp)
 8956        # H ^ 5
 8957        # ghash_gfmul_red_avx
 8958        vpshufd	$0x4e, %xmm0, %xmm9
 8959        vpshufd	$0x4e, %xmm1, %xmm10
 8960        vpclmulqdq	$0x11, %xmm0, %xmm1, %xmm11
 8961        vpclmulqdq	$0x00, %xmm0, %xmm1, %xmm8
 8962        vpxor	%xmm0, %xmm9, %xmm9
 8963        vpxor	%xmm1, %xmm10, %xmm10
 8964        vpclmulqdq	$0x00, %xmm10, %xmm9, %xmm9
 8965        vpxor	%xmm8, %xmm9, %xmm9
 8966        vpxor	%xmm11, %xmm9, %xmm9
 8967        vpslldq	$8, %xmm9, %xmm10
 8968        vpsrldq	$8, %xmm9, %xmm9
 8969        vpxor	%xmm10, %xmm8, %xmm8
 8970        vpxor	%xmm9, %xmm11, %xmm7
 8971        vpslld	$31, %xmm8, %xmm12
 8972        vpslld	$30, %xmm8, %xmm13
 8973        vpslld	$25, %xmm8, %xmm14
 8974        vpxor	%xmm13, %xmm12, %xmm12
 8975        vpxor	%xmm14, %xmm12, %xmm12
 8976        vpsrldq	$4, %xmm12, %xmm13
 8977        vpslldq	$12, %xmm12, %xmm12
 8978        vpxor	%xmm12, %xmm8, %xmm8
 8979        vpsrld	$0x01, %xmm8, %xmm14
 8980        vpsrld	$2, %xmm8, %xmm10
 8981        vpsrld	$7, %xmm8, %xmm9
 8982        vpxor	%xmm10, %xmm14, %xmm14
 8983        vpxor	%xmm9, %xmm14, %xmm14
 8984        vpxor	%xmm13, %xmm14, %xmm14
 8985        vpxor	%xmm8, %xmm14, %xmm14
 8986        vpxor	%xmm14, %xmm7, %xmm7
 8987        vmovdqu	%xmm7, 64(%rsp)
 8988        # H ^ 6
 8989        vpclmulqdq	$0x00, %xmm1, %xmm1, %xmm8
 8990        vpclmulqdq	$0x11, %xmm1, %xmm1, %xmm7
 8991        vpslld	$31, %xmm8, %xmm12
 8992        vpslld	$30, %xmm8, %xmm13
 8993        vpslld	$25, %xmm8, %xmm14
 8994        vpxor	%xmm13, %xmm12, %xmm12
 8995        vpxor	%xmm14, %xmm12, %xmm12
 8996        vpsrldq	$4, %xmm12, %xmm13
 8997        vpslldq	$12, %xmm12, %xmm12
 8998        vpxor	%xmm12, %xmm8, %xmm8
 8999        vpsrld	$0x01, %xmm8, %xmm14
 9000        vpsrld	$2, %xmm8, %xmm10
 9001        vpsrld	$7, %xmm8, %xmm9
 9002        vpxor	%xmm10, %xmm14, %xmm14
 9003        vpxor	%xmm9, %xmm14, %xmm14
 9004        vpxor	%xmm13, %xmm14, %xmm14
 9005        vpxor	%xmm8, %xmm14, %xmm14
 9006        vpxor	%xmm14, %xmm7, %xmm7
 9007        vmovdqu	%xmm7, 80(%rsp)
 9008        # H ^ 7
 9009        # ghash_gfmul_red_avx
 9010        vpshufd	$0x4e, %xmm1, %xmm9
 9011        vpshufd	$0x4e, %xmm3, %xmm10
 9012        vpclmulqdq	$0x11, %xmm1, %xmm3, %xmm11
 9013        vpclmulqdq	$0x00, %xmm1, %xmm3, %xmm8
 9014        vpxor	%xmm1, %xmm9, %xmm9
 9015        vpxor	%xmm3, %xmm10, %xmm10
 9016        vpclmulqdq	$0x00, %xmm10, %xmm9, %xmm9
 9017        vpxor	%xmm8, %xmm9, %xmm9
 9018        vpxor	%xmm11, %xmm9, %xmm9
 9019        vpslldq	$8, %xmm9, %xmm10
 9020        vpsrldq	$8, %xmm9, %xmm9
 9021        vpxor	%xmm10, %xmm8, %xmm8
 9022        vpxor	%xmm9, %xmm11, %xmm7
 9023        vpslld	$31, %xmm8, %xmm12
 9024        vpslld	$30, %xmm8, %xmm13
 9025        vpslld	$25, %xmm8, %xmm14
 9026        vpxor	%xmm13, %xmm12, %xmm12
 9027        vpxor	%xmm14, %xmm12, %xmm12
 9028        vpsrldq	$4, %xmm12, %xmm13
 9029        vpslldq	$12, %xmm12, %xmm12
 9030        vpxor	%xmm12, %xmm8, %xmm8
 9031        vpsrld	$0x01, %xmm8, %xmm14
 9032        vpsrld	$2, %xmm8, %xmm10
 9033        vpsrld	$7, %xmm8, %xmm9
 9034        vpxor	%xmm10, %xmm14, %xmm14
 9035        vpxor	%xmm9, %xmm14, %xmm14
 9036        vpxor	%xmm13, %xmm14, %xmm14
 9037        vpxor	%xmm8, %xmm14, %xmm14
 9038        vpxor	%xmm14, %xmm7, %xmm7
 9039        vmovdqu	%xmm7, 96(%rsp)
 9040        # H ^ 8
 9041        vpclmulqdq	$0x00, %xmm3, %xmm3, %xmm8
 9042        vpclmulqdq	$0x11, %xmm3, %xmm3, %xmm7
 9043        vpslld	$31, %xmm8, %xmm12
 9044        vpslld	$30, %xmm8, %xmm13
 9045        vpslld	$25, %xmm8, %xmm14
 9046        vpxor	%xmm13, %xmm12, %xmm12
 9047        vpxor	%xmm14, %xmm12, %xmm12
 9048        vpsrldq	$4, %xmm12, %xmm13
 9049        vpslldq	$12, %xmm12, %xmm12
 9050        vpxor	%xmm12, %xmm8, %xmm8
 9051        vpsrld	$0x01, %xmm8, %xmm14
 9052        vpsrld	$2, %xmm8, %xmm10
 9053        vpsrld	$7, %xmm8, %xmm9
 9054        vpxor	%xmm10, %xmm14, %xmm14
 9055        vpxor	%xmm9, %xmm14, %xmm14
 9056        vpxor	%xmm13, %xmm14, %xmm14
 9057        vpxor	%xmm8, %xmm14, %xmm14
 9058        vpxor	%xmm14, %xmm7, %xmm7
 9059        vmovdqu	%xmm7, 112(%rsp)
 9060L_AES_GCM_decrypt_avx1_ghash_128:
 9061        leaq	(%rdi,%rbx,1), %rcx
 9062        leaq	(%rsi,%rbx,1), %rdx
 9063        vmovdqu	128(%rsp), %xmm0
 9064        vmovdqa	L_avx1_aes_gcm_bswap_epi64(%rip), %xmm1
 9065        vpshufb	%xmm1, %xmm0, %xmm8
 9066        vpaddd	L_avx1_aes_gcm_one(%rip), %xmm0, %xmm9
 9067        vpshufb	%xmm1, %xmm9, %xmm9
 9068        vpaddd	L_avx1_aes_gcm_two(%rip), %xmm0, %xmm10
 9069        vpshufb	%xmm1, %xmm10, %xmm10
 9070        vpaddd	L_avx1_aes_gcm_three(%rip), %xmm0, %xmm11
 9071        vpshufb	%xmm1, %xmm11, %xmm11
 9072        vpaddd	L_avx1_aes_gcm_four(%rip), %xmm0, %xmm12
 9073        vpshufb	%xmm1, %xmm12, %xmm12
 9074        vpaddd	L_avx1_aes_gcm_five(%rip), %xmm0, %xmm13
 9075        vpshufb	%xmm1, %xmm13, %xmm13
 9076        vpaddd	L_avx1_aes_gcm_six(%rip), %xmm0, %xmm14
 9077        vpshufb	%xmm1, %xmm14, %xmm14
 9078        vpaddd	L_avx1_aes_gcm_seven(%rip), %xmm0, %xmm15
 9079        vpshufb	%xmm1, %xmm15, %xmm15
 9080        vpaddd	L_avx1_aes_gcm_eight(%rip), %xmm0, %xmm0
 9081        vmovdqa	(%r15), %xmm7
 9082        vmovdqu	%xmm0, 128(%rsp)
 9083        vpxor	%xmm7, %xmm8, %xmm8
 9084        vpxor	%xmm7, %xmm9, %xmm9
 9085        vpxor	%xmm7, %xmm10, %xmm10
 9086        vpxor	%xmm7, %xmm11, %xmm11
 9087        vpxor	%xmm7, %xmm12, %xmm12
 9088        vpxor	%xmm7, %xmm13, %xmm13
 9089        vpxor	%xmm7, %xmm14, %xmm14
 9090        vpxor	%xmm7, %xmm15, %xmm15
 9091        vmovdqu	112(%rsp), %xmm7
 9092        vmovdqu	(%rcx), %xmm0
 9093        vaesenc	16(%r15), %xmm8, %xmm8
 9094        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0
 9095        vpxor	%xmm2, %xmm0, %xmm0
 9096        vpshufd	$0x4e, %xmm7, %xmm1
 9097        vpshufd	$0x4e, %xmm0, %xmm5
 9098        vpxor	%xmm7, %xmm1, %xmm1
 9099        vpxor	%xmm0, %xmm5, %xmm5
 9100        vpclmulqdq	$0x11, %xmm7, %xmm0, %xmm3
 9101        vaesenc	16(%r15), %xmm9, %xmm9
 9102        vaesenc	16(%r15), %xmm10, %xmm10
 9103        vpclmulqdq	$0x00, %xmm7, %xmm0, %xmm2
 9104        vaesenc	16(%r15), %xmm11, %xmm11
 9105        vaesenc	16(%r15), %xmm12, %xmm12
 9106        vpclmulqdq	$0x00, %xmm5, %xmm1, %xmm1
 9107        vaesenc	16(%r15), %xmm13, %xmm13
 9108        vaesenc	16(%r15), %xmm14, %xmm14
 9109        vaesenc	16(%r15), %xmm15, %xmm15
 9110        vpxor	%xmm2, %xmm1, %xmm1
 9111        vpxor	%xmm3, %xmm1, %xmm1
 9112        vmovdqu	96(%rsp), %xmm7
 9113        vmovdqu	16(%rcx), %xmm0
 9114        vpshufd	$0x4e, %xmm7, %xmm4
 9115        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0
 9116        vaesenc	32(%r15), %xmm8, %xmm8
 9117        vpxor	%xmm7, %xmm4, %xmm4
 9118        vpshufd	$0x4e, %xmm0, %xmm5
 9119        vpxor	%xmm0, %xmm5, %xmm5
 9120        vpclmulqdq	$0x11, %xmm7, %xmm0, %xmm6
 9121        vaesenc	32(%r15), %xmm9, %xmm9
 9122        vaesenc	32(%r15), %xmm10, %xmm10
 9123        vpclmulqdq	$0x00, %xmm7, %xmm0, %xmm7
 9124        vaesenc	32(%r15), %xmm11, %xmm11
 9125        vaesenc	32(%r15), %xmm12, %xmm12
 9126        vpclmulqdq	$0x00, %xmm5, %xmm4, %xmm4
 9127        vaesenc	32(%r15), %xmm13, %xmm13
 9128        vaesenc	32(%r15), %xmm14, %xmm14
 9129        vaesenc	32(%r15), %xmm15, %xmm15
 9130        vpxor	%xmm7, %xmm1, %xmm1
 9131        vpxor	%xmm7, %xmm2, %xmm2
 9132        vpxor	%xmm6, %xmm1, %xmm1
 9133        vpxor	%xmm6, %xmm3, %xmm3
 9134        vpxor	%xmm4, %xmm1, %xmm1
 9135        vmovdqu	80(%rsp), %xmm7
 9136        vmovdqu	32(%rcx), %xmm0
 9137        vpshufd	$0x4e, %xmm7, %xmm4
 9138        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0
 9139        vaesenc	48(%r15), %xmm8, %xmm8
 9140        vpxor	%xmm7, %xmm4, %xmm4
 9141        vpshufd	$0x4e, %xmm0, %xmm5
 9142        vpxor	%xmm0, %xmm5, %xmm5
 9143        vpclmulqdq	$0x11, %xmm7, %xmm0, %xmm6
 9144        vaesenc	48(%r15), %xmm9, %xmm9
 9145        vaesenc	48(%r15), %xmm10, %xmm10
 9146        vpclmulqdq	$0x00, %xmm7, %xmm0, %xmm7
 9147        vaesenc	48(%r15), %xmm11, %xmm11
 9148        vaesenc	48(%r15), %xmm12, %xmm12
 9149        vpclmulqdq	$0x00, %xmm5, %xmm4, %xmm4
 9150        vaesenc	48(%r15), %xmm13, %xmm13
 9151        vaesenc	48(%r15), %xmm14, %xmm14
 9152        vaesenc	48(%r15), %xmm15, %xmm15
 9153        vpxor	%xmm7, %xmm1, %xmm1
 9154        vpxor	%xmm7, %xmm2, %xmm2
 9155        vpxor	%xmm6, %xmm1, %xmm1
 9156        vpxor	%xmm6, %xmm3, %xmm3
 9157        vpxor	%xmm4, %xmm1, %xmm1
 9158        vmovdqu	64(%rsp), %xmm7
 9159        vmovdqu	48(%rcx), %xmm0
 9160        vpshufd	$0x4e, %xmm7, %xmm4
 9161        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0
 9162        vaesenc	64(%r15), %xmm8, %xmm8
 9163        vpxor	%xmm7, %xmm4, %xmm4
 9164        vpshufd	$0x4e, %xmm0, %xmm5
 9165        vpxor	%xmm0, %xmm5, %xmm5
 9166        vpclmulqdq	$0x11, %xmm7, %xmm0, %xmm6
 9167        vaesenc	64(%r15), %xmm9, %xmm9
 9168        vaesenc	64(%r15), %xmm10, %xmm10
 9169        vpclmulqdq	$0x00, %xmm7, %xmm0, %xmm7
 9170        vaesenc	64(%r15), %xmm11, %xmm11
 9171        vaesenc	64(%r15), %xmm12, %xmm12
 9172        vpclmulqdq	$0x00, %xmm5, %xmm4, %xmm4
 9173        vaesenc	64(%r15), %xmm13, %xmm13
 9174        vaesenc	64(%r15), %xmm14, %xmm14
 9175        vaesenc	64(%r15), %xmm15, %xmm15
 9176        vpxor	%xmm7, %xmm1, %xmm1
 9177        vpxor	%xmm7, %xmm2, %xmm2
 9178        vpxor	%xmm6, %xmm1, %xmm1
 9179        vpxor	%xmm6, %xmm3, %xmm3
 9180        vpxor	%xmm4, %xmm1, %xmm1
 9181        vmovdqu	48(%rsp), %xmm7
 9182        vmovdqu	64(%rcx), %xmm0
 9183        vpshufd	$0x4e, %xmm7, %xmm4
 9184        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0
 9185        vaesenc	80(%r15), %xmm8, %xmm8
 9186        vpxor	%xmm7, %xmm4, %xmm4
 9187        vpshufd	$0x4e, %xmm0, %xmm5
 9188        vpxor	%xmm0, %xmm5, %xmm5
 9189        vpclmulqdq	$0x11, %xmm7, %xmm0, %xmm6
 9190        vaesenc	80(%r15), %xmm9, %xmm9
 9191        vaesenc	80(%r15), %xmm10, %xmm10
 9192        vpclmulqdq	$0x00, %xmm7, %xmm0, %xmm7
 9193        vaesenc	80(%r15), %xmm11, %xmm11
 9194        vaesenc	80(%r15), %xmm12, %xmm12
 9195        vpclmulqdq	$0x00, %xmm5, %xmm4, %xmm4
 9196        vaesenc	80(%r15), %xmm13, %xmm13
 9197        vaesenc	80(%r15), %xmm14, %xmm14
 9198        vaesenc	80(%r15), %xmm15, %xmm15
 9199        vpxor	%xmm7, %xmm1, %xmm1
 9200        vpxor	%xmm7, %xmm2, %xmm2
 9201        vpxor	%xmm6, %xmm1, %xmm1
 9202        vpxor	%xmm6, %xmm3, %xmm3
 9203        vpxor	%xmm4, %xmm1, %xmm1
 9204        vmovdqu	32(%rsp), %xmm7
 9205        vmovdqu	80(%rcx), %xmm0
 9206        vpshufd	$0x4e, %xmm7, %xmm4
 9207        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0
 9208        vaesenc	96(%r15), %xmm8, %xmm8
 9209        vpxor	%xmm7, %xmm4, %xmm4
 9210        vpshufd	$0x4e, %xmm0, %xmm5
 9211        vpxor	%xmm0, %xmm5, %xmm5
 9212        vpclmulqdq	$0x11, %xmm7, %xmm0, %xmm6
 9213        vaesenc	96(%r15), %xmm9, %xmm9
 9214        vaesenc	96(%r15), %xmm10, %xmm10
 9215        vpclmulqdq	$0x00, %xmm7, %xmm0, %xmm7
 9216        vaesenc	96(%r15), %xmm11, %xmm11
 9217        vaesenc	96(%r15), %xmm12, %xmm12
 9218        vpclmulqdq	$0x00, %xmm5, %xmm4, %xmm4
 9219        vaesenc	96(%r15), %xmm13, %xmm13
 9220        vaesenc	96(%r15), %xmm14, %xmm14
 9221        vaesenc	96(%r15), %xmm15, %xmm15
 9222        vpxor	%xmm7, %xmm1, %xmm1
 9223        vpxor	%xmm7, %xmm2, %xmm2
 9224        vpxor	%xmm6, %xmm1, %xmm1
 9225        vpxor	%xmm6, %xmm3, %xmm3
 9226        vpxor	%xmm4, %xmm1, %xmm1
 9227        vmovdqu	16(%rsp), %xmm7
 9228        vmovdqu	96(%rcx), %xmm0
 9229        vpshufd	$0x4e, %xmm7, %xmm4
 9230        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0
 9231        vaesenc	112(%r15), %xmm8, %xmm8
 9232        vpxor	%xmm7, %xmm4, %xmm4
 9233        vpshufd	$0x4e, %xmm0, %xmm5
 9234        vpxor	%xmm0, %xmm5, %xmm5
 9235        vpclmulqdq	$0x11, %xmm7, %xmm0, %xmm6
 9236        vaesenc	112(%r15), %xmm9, %xmm9
 9237        vaesenc	112(%r15), %xmm10, %xmm10
 9238        vpclmulqdq	$0x00, %xmm7, %xmm0, %xmm7
 9239        vaesenc	112(%r15), %xmm11, %xmm11
 9240        vaesenc	112(%r15), %xmm12, %xmm12
 9241        vpclmulqdq	$0x00, %xmm5, %xmm4, %xmm4
 9242        vaesenc	112(%r15), %xmm13, %xmm13
 9243        vaesenc	112(%r15), %xmm14, %xmm14
 9244        vaesenc	112(%r15), %xmm15, %xmm15
 9245        vpxor	%xmm7, %xmm1, %xmm1
 9246        vpxor	%xmm7, %xmm2, %xmm2
 9247        vpxor	%xmm6, %xmm1, %xmm1
 9248        vpxor	%xmm6, %xmm3, %xmm3
 9249        vpxor	%xmm4, %xmm1, %xmm1
 9250        vmovdqu	(%rsp), %xmm7
 9251        vmovdqu	112(%rcx), %xmm0
 9252        vpshufd	$0x4e, %xmm7, %xmm4
 9253        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0
 9254        vaesenc	128(%r15), %xmm8, %xmm8
 9255        vpxor	%xmm7, %xmm4, %xmm4
 9256        vpshufd	$0x4e, %xmm0, %xmm5
 9257        vpxor	%xmm0, %xmm5, %xmm5
 9258        vpclmulqdq	$0x11, %xmm7, %xmm0, %xmm6
 9259        vaesenc	128(%r15), %xmm9, %xmm9
 9260        vaesenc	128(%r15), %xmm10, %xmm10
 9261        vpclmulqdq	$0x00, %xmm7, %xmm0, %xmm7
 9262        vaesenc	128(%r15), %xmm11, %xmm11
 9263        vaesenc	128(%r15), %xmm12, %xmm12
 9264        vpclmulqdq	$0x00, %xmm5, %xmm4, %xmm4
 9265        vaesenc	128(%r15), %xmm13, %xmm13
 9266        vaesenc	128(%r15), %xmm14, %xmm14
 9267        vaesenc	128(%r15), %xmm15, %xmm15
 9268        vpxor	%xmm7, %xmm1, %xmm1
 9269        vpxor	%xmm7, %xmm2, %xmm2
 9270        vpxor	%xmm6, %xmm1, %xmm1
 9271        vpxor	%xmm6, %xmm3, %xmm3
 9272        vpxor	%xmm4, %xmm1, %xmm1
 9273        vpslldq	$8, %xmm1, %xmm5
 9274        vpsrldq	$8, %xmm1, %xmm1
 9275        vaesenc	144(%r15), %xmm8, %xmm8
 9276        vpxor	%xmm5, %xmm2, %xmm2
 9277        vpxor	%xmm1, %xmm3, %xmm3
 9278        vaesenc	144(%r15), %xmm9, %xmm9
 9279        vpslld	$31, %xmm2, %xmm7
 9280        vpslld	$30, %xmm2, %xmm4
 9281        vpslld	$25, %xmm2, %xmm5
 9282        vaesenc	144(%r15), %xmm10, %xmm10
 9283        vpxor	%xmm4, %xmm7, %xmm7
 9284        vpxor	%xmm5, %xmm7, %xmm7
 9285        vaesenc	144(%r15), %xmm11, %xmm11
 9286        vpsrldq	$4, %xmm7, %xmm4
 9287        vpslldq	$12, %xmm7, %xmm7
 9288        vaesenc	144(%r15), %xmm12, %xmm12
 9289        vpxor	%xmm7, %xmm2, %xmm2
 9290        vpsrld	$0x01, %xmm2, %xmm5
 9291        vaesenc	144(%r15), %xmm13, %xmm13
 9292        vpsrld	$2, %xmm2, %xmm1
 9293        vpsrld	$7, %xmm2, %xmm0
 9294        vaesenc	144(%r15), %xmm14, %xmm14
 9295        vpxor	%xmm1, %xmm5, %xmm5
 9296        vpxor	%xmm0, %xmm5, %xmm5
 9297        vaesenc	144(%r15), %xmm15, %xmm15
 9298        vpxor	%xmm4, %xmm5, %xmm5
 9299        vpxor	%xmm5, %xmm2, %xmm2
 9300        vpxor	%xmm3, %xmm2, %xmm2
 9301        cmpl	$11, %r10d
 9302        vmovdqa	160(%r15), %xmm7
 9303        jl	L_AES_GCM_decrypt_avx1_aesenc_128_ghash_avx_done
 9304        vaesenc	%xmm7, %xmm8, %xmm8
 9305        vaesenc	%xmm7, %xmm9, %xmm9
 9306        vaesenc	%xmm7, %xmm10, %xmm10
 9307        vaesenc	%xmm7, %xmm11, %xmm11
 9308        vaesenc	%xmm7, %xmm12, %xmm12
 9309        vaesenc	%xmm7, %xmm13, %xmm13
 9310        vaesenc	%xmm7, %xmm14, %xmm14
 9311        vaesenc	%xmm7, %xmm15, %xmm15
 9312        vmovdqa	176(%r15), %xmm7
 9313        vaesenc	%xmm7, %xmm8, %xmm8
 9314        vaesenc	%xmm7, %xmm9, %xmm9
 9315        vaesenc	%xmm7, %xmm10, %xmm10
 9316        vaesenc	%xmm7, %xmm11, %xmm11
 9317        vaesenc	%xmm7, %xmm12, %xmm12
 9318        vaesenc	%xmm7, %xmm13, %xmm13
 9319        vaesenc	%xmm7, %xmm14, %xmm14
 9320        vaesenc	%xmm7, %xmm15, %xmm15
 9321        cmpl	$13, %r10d
 9322        vmovdqa	192(%r15), %xmm7
 9323        jl	L_AES_GCM_decrypt_avx1_aesenc_128_ghash_avx_done
 9324        vaesenc	%xmm7, %xmm8, %xmm8
 9325        vaesenc	%xmm7, %xmm9, %xmm9
 9326        vaesenc	%xmm7, %xmm10, %xmm10
 9327        vaesenc	%xmm7, %xmm11, %xmm11
 9328        vaesenc	%xmm7, %xmm12, %xmm12
 9329        vaesenc	%xmm7, %xmm13, %xmm13
 9330        vaesenc	%xmm7, %xmm14, %xmm14
 9331        vaesenc	%xmm7, %xmm15, %xmm15
 9332        vmovdqa	208(%r15), %xmm7
 9333        vaesenc	%xmm7, %xmm8, %xmm8
 9334        vaesenc	%xmm7, %xmm9, %xmm9
 9335        vaesenc	%xmm7, %xmm10, %xmm10
 9336        vaesenc	%xmm7, %xmm11, %xmm11
 9337        vaesenc	%xmm7, %xmm12, %xmm12
 9338        vaesenc	%xmm7, %xmm13, %xmm13
 9339        vaesenc	%xmm7, %xmm14, %xmm14
 9340        vaesenc	%xmm7, %xmm15, %xmm15
 9341        vmovdqa	224(%r15), %xmm7
 9342L_AES_GCM_decrypt_avx1_aesenc_128_ghash_avx_done:
 9343        vaesenclast	%xmm7, %xmm8, %xmm8
 9344        vaesenclast	%xmm7, %xmm9, %xmm9
 9345        vmovdqu	(%rcx), %xmm0
 9346        vmovdqu	16(%rcx), %xmm1
 9347        vpxor	%xmm0, %xmm8, %xmm8
 9348        vpxor	%xmm1, %xmm9, %xmm9
 9349        vmovdqu	%xmm8, (%rdx)
 9350        vmovdqu	%xmm9, 16(%rdx)
 9351        vaesenclast	%xmm7, %xmm10, %xmm10
 9352        vaesenclast	%xmm7, %xmm11, %xmm11
 9353        vmovdqu	32(%rcx), %xmm0
 9354        vmovdqu	48(%rcx), %xmm1
 9355        vpxor	%xmm0, %xmm10, %xmm10
 9356        vpxor	%xmm1, %xmm11, %xmm11
 9357        vmovdqu	%xmm10, 32(%rdx)
 9358        vmovdqu	%xmm11, 48(%rdx)
 9359        vaesenclast	%xmm7, %xmm12, %xmm12
 9360        vaesenclast	%xmm7, %xmm13, %xmm13
 9361        vmovdqu	64(%rcx), %xmm0
 9362        vmovdqu	80(%rcx), %xmm1
 9363        vpxor	%xmm0, %xmm12, %xmm12
 9364        vpxor	%xmm1, %xmm13, %xmm13
 9365        vmovdqu	%xmm12, 64(%rdx)
 9366        vmovdqu	%xmm13, 80(%rdx)
 9367        vaesenclast	%xmm7, %xmm14, %xmm14
 9368        vaesenclast	%xmm7, %xmm15, %xmm15
 9369        vmovdqu	96(%rcx), %xmm0
 9370        vmovdqu	112(%rcx), %xmm1
 9371        vpxor	%xmm0, %xmm14, %xmm14
 9372        vpxor	%xmm1, %xmm15, %xmm15
 9373        vmovdqu	%xmm14, 96(%rdx)
 9374        vmovdqu	%xmm15, 112(%rdx)
 9375        addl	$0x80, %ebx
 9376        cmpl	%r13d, %ebx
 9377        jl	L_AES_GCM_decrypt_avx1_ghash_128
 9378        vmovdqa	%xmm2, %xmm6
 9379        vmovdqu	(%rsp), %xmm5
 9380L_AES_GCM_decrypt_avx1_done_128:
 9381        movl	%r9d, %edx
 9382        cmpl	%edx, %ebx
 9383        jge	L_AES_GCM_decrypt_avx1_done_dec
 9384        movl	%r9d, %r13d
 9385        andl	$0xfffffff0, %r13d
 9386        cmpl	%r13d, %ebx
 9387        jge	L_AES_GCM_decrypt_avx1_last_block_done
 9388L_AES_GCM_decrypt_avx1_last_block_start:
 9389        vmovdqu	(%rdi,%rbx,1), %xmm13
 9390        vmovdqa	%xmm5, %xmm0
 9391        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm13, %xmm1
 9392        vpxor	%xmm6, %xmm1, %xmm1
 9393        vmovdqu	128(%rsp), %xmm9
 9394        vpshufb	L_avx1_aes_gcm_bswap_epi64(%rip), %xmm9, %xmm8
 9395        vpaddd	L_avx1_aes_gcm_one(%rip), %xmm9, %xmm9
 9396        vmovdqu	%xmm9, 128(%rsp)
 9397        vpxor	(%r15), %xmm8, %xmm8
 9398        vpclmulqdq	$16, %xmm0, %xmm1, %xmm10
 9399        vaesenc	16(%r15), %xmm8, %xmm8
 9400        vaesenc	32(%r15), %xmm8, %xmm8
 9401        vpclmulqdq	$0x01, %xmm0, %xmm1, %xmm11
 9402        vaesenc	48(%r15), %xmm8, %xmm8
 9403        vaesenc	64(%r15), %xmm8, %xmm8
 9404        vpclmulqdq	$0x00, %xmm0, %xmm1, %xmm12
 9405        vaesenc	80(%r15), %xmm8, %xmm8
 9406        vpclmulqdq	$0x11, %xmm0, %xmm1, %xmm1
 9407        vaesenc	96(%r15), %xmm8, %xmm8
 9408        vpxor	%xmm11, %xmm10, %xmm10
 9409        vpslldq	$8, %xmm10, %xmm2
 9410        vpsrldq	$8, %xmm10, %xmm10
 9411        vaesenc	112(%r15), %xmm8, %xmm8
 9412        vpxor	%xmm12, %xmm2, %xmm2
 9413        vpxor	%xmm10, %xmm1, %xmm3
 9414        vmovdqa	L_avx1_aes_gcm_mod2_128(%rip), %xmm0
 9415        vpclmulqdq	$16, %xmm0, %xmm2, %xmm11
 9416        vaesenc	128(%r15), %xmm8, %xmm8
 9417        vpshufd	$0x4e, %xmm2, %xmm10
 9418        vpxor	%xmm11, %xmm10, %xmm10
 9419        vpclmulqdq	$16, %xmm0, %xmm10, %xmm11
 9420        vaesenc	144(%r15), %xmm8, %xmm8
 9421        vpshufd	$0x4e, %xmm10, %xmm10
 9422        vpxor	%xmm11, %xmm10, %xmm10
 9423        vpxor	%xmm3, %xmm10, %xmm6
 9424        cmpl	$11, %r10d
 9425        vmovdqa	160(%r15), %xmm9
 9426        jl	L_AES_GCM_decrypt_avx1_aesenc_gfmul_last
 9427        vaesenc	%xmm9, %xmm8, %xmm8
 9428        vaesenc	176(%r15), %xmm8, %xmm8
 9429        cmpl	$13, %r10d
 9430        vmovdqa	192(%r15), %xmm9
 9431        jl	L_AES_GCM_decrypt_avx1_aesenc_gfmul_last
 9432        vaesenc	%xmm9, %xmm8, %xmm8
 9433        vaesenc	208(%r15), %xmm8, %xmm8
 9434        vmovdqa	224(%r15), %xmm9
 9435L_AES_GCM_decrypt_avx1_aesenc_gfmul_last:
 9436        vaesenclast	%xmm9, %xmm8, %xmm8
 9437        vmovdqa	%xmm13, %xmm0
 9438        vpxor	%xmm0, %xmm8, %xmm8
 9439        vmovdqu	%xmm8, (%rsi,%rbx,1)
 9440        addl	$16, %ebx
 9441        cmpl	%r13d, %ebx
 9442        jl	L_AES_GCM_decrypt_avx1_last_block_start
 9443L_AES_GCM_decrypt_avx1_last_block_done:
 9444        movl	%r9d, %ecx
 9445        movl	%ecx, %edx
 9446        andl	$15, %ecx
 9447        jz	L_AES_GCM_decrypt_avx1_aesenc_last15_dec_avx_done
 9448        vmovdqu	128(%rsp), %xmm4
 9449        vpshufb	L_avx1_aes_gcm_bswap_epi64(%rip), %xmm4, %xmm4
 9450        vpxor	(%r15), %xmm4, %xmm4
 9451        vaesenc	16(%r15), %xmm4, %xmm4
 9452        vaesenc	32(%r15), %xmm4, %xmm4
 9453        vaesenc	48(%r15), %xmm4, %xmm4
 9454        vaesenc	64(%r15), %xmm4, %xmm4
 9455        vaesenc	80(%r15), %xmm4, %xmm4
 9456        vaesenc	96(%r15), %xmm4, %xmm4
 9457        vaesenc	112(%r15), %xmm4, %xmm4
 9458        vaesenc	128(%r15), %xmm4, %xmm4
 9459        vaesenc	144(%r15), %xmm4, %xmm4
 9460        cmpl	$11, %r10d
 9461        vmovdqa	160(%r15), %xmm9
 9462        jl	L_AES_GCM_decrypt_avx1_aesenc_last15_dec_avx_aesenc_avx_last
 9463        vaesenc	%xmm9, %xmm4, %xmm4
 9464        vaesenc	176(%r15), %xmm4, %xmm4
 9465        cmpl	$13, %r10d
 9466        vmovdqa	192(%r15), %xmm9
 9467        jl	L_AES_GCM_decrypt_avx1_aesenc_last15_dec_avx_aesenc_avx_last
 9468        vaesenc	%xmm9, %xmm4, %xmm4
 9469        vaesenc	208(%r15), %xmm4, %xmm4
 9470        vmovdqa	224(%r15), %xmm9
 9471L_AES_GCM_decrypt_avx1_aesenc_last15_dec_avx_aesenc_avx_last:
 9472        vaesenclast	%xmm9, %xmm4, %xmm4
 9473        subq	$32, %rsp
 9474        xorl	%ecx, %ecx
 9475        vmovdqu	%xmm4, (%rsp)
 9476        vpxor	%xmm0, %xmm0, %xmm0
 9477        vmovdqu	%xmm0, 16(%rsp)
 9478L_AES_GCM_decrypt_avx1_aesenc_last15_dec_avx_loop:
 9479        movzbl	(%rdi,%rbx,1), %r13d
 9480        movb	%r13b, 16(%rsp,%rcx,1)
 9481        xorb	(%rsp,%rcx,1), %r13b
 9482        movb	%r13b, (%rsi,%rbx,1)
 9483        incl	%ebx
 9484        incl	%ecx
 9485        cmpl	%edx, %ebx
 9486        jl	L_AES_GCM_decrypt_avx1_aesenc_last15_dec_avx_loop
 9487        vmovdqu	16(%rsp), %xmm4
 9488        addq	$32, %rsp
 9489        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm4, %xmm4
 9490        vpxor	%xmm4, %xmm6, %xmm6
 9491        # ghash_gfmul_red_avx
 9492        vpshufd	$0x4e, %xmm5, %xmm9
 9493        vpshufd	$0x4e, %xmm6, %xmm10
 9494        vpclmulqdq	$0x11, %xmm5, %xmm6, %xmm11
 9495        vpclmulqdq	$0x00, %xmm5, %xmm6, %xmm8
 9496        vpxor	%xmm5, %xmm9, %xmm9
 9497        vpxor	%xmm6, %xmm10, %xmm10
 9498        vpclmulqdq	$0x00, %xmm10, %xmm9, %xmm9
 9499        vpxor	%xmm8, %xmm9, %xmm9
 9500        vpxor	%xmm11, %xmm9, %xmm9
 9501        vpslldq	$8, %xmm9, %xmm10
 9502        vpsrldq	$8, %xmm9, %xmm9
 9503        vpxor	%xmm10, %xmm8, %xmm8
 9504        vpxor	%xmm9, %xmm11, %xmm6
 9505        vpslld	$31, %xmm8, %xmm12
 9506        vpslld	$30, %xmm8, %xmm13
 9507        vpslld	$25, %xmm8, %xmm14
 9508        vpxor	%xmm13, %xmm12, %xmm12
 9509        vpxor	%xmm14, %xmm12, %xmm12
 9510        vpsrldq	$4, %xmm12, %xmm13
 9511        vpslldq	$12, %xmm12, %xmm12
 9512        vpxor	%xmm12, %xmm8, %xmm8
 9513        vpsrld	$0x01, %xmm8, %xmm14
 9514        vpsrld	$2, %xmm8, %xmm10
 9515        vpsrld	$7, %xmm8, %xmm9
 9516        vpxor	%xmm10, %xmm14, %xmm14
 9517        vpxor	%xmm9, %xmm14, %xmm14
 9518        vpxor	%xmm13, %xmm14, %xmm14
 9519        vpxor	%xmm8, %xmm14, %xmm14
 9520        vpxor	%xmm14, %xmm6, %xmm6
 9521L_AES_GCM_decrypt_avx1_aesenc_last15_dec_avx_done:
 9522L_AES_GCM_decrypt_avx1_done_dec:
 9523        movl	%r9d, %edx
 9524        movl	%r11d, %ecx
 9525        shlq	$3, %rdx
 9526        shlq	$3, %rcx
 9527        vmovq	%rdx, %xmm0
 9528        vmovq	%rcx, %xmm1
 9529        vpunpcklqdq	%xmm1, %xmm0, %xmm0
 9530        vpxor	%xmm0, %xmm6, %xmm6
 9531        # ghash_gfmul_red_avx
 9532        vpshufd	$0x4e, %xmm5, %xmm9
 9533        vpshufd	$0x4e, %xmm6, %xmm10
 9534        vpclmulqdq	$0x11, %xmm5, %xmm6, %xmm11
 9535        vpclmulqdq	$0x00, %xmm5, %xmm6, %xmm8
 9536        vpxor	%xmm5, %xmm9, %xmm9
 9537        vpxor	%xmm6, %xmm10, %xmm10
 9538        vpclmulqdq	$0x00, %xmm10, %xmm9, %xmm9
 9539        vpxor	%xmm8, %xmm9, %xmm9
 9540        vpxor	%xmm11, %xmm9, %xmm9
 9541        vpslldq	$8, %xmm9, %xmm10
 9542        vpsrldq	$8, %xmm9, %xmm9
 9543        vpxor	%xmm10, %xmm8, %xmm8
 9544        vpxor	%xmm9, %xmm11, %xmm6
 9545        vpslld	$31, %xmm8, %xmm12
 9546        vpslld	$30, %xmm8, %xmm13
 9547        vpslld	$25, %xmm8, %xmm14
 9548        vpxor	%xmm13, %xmm12, %xmm12
 9549        vpxor	%xmm14, %xmm12, %xmm12
 9550        vpsrldq	$4, %xmm12, %xmm13
 9551        vpslldq	$12, %xmm12, %xmm12
 9552        vpxor	%xmm12, %xmm8, %xmm8
 9553        vpsrld	$0x01, %xmm8, %xmm14
 9554        vpsrld	$2, %xmm8, %xmm10
 9555        vpsrld	$7, %xmm8, %xmm9
 9556        vpxor	%xmm10, %xmm14, %xmm14
 9557        vpxor	%xmm9, %xmm14, %xmm14
 9558        vpxor	%xmm13, %xmm14, %xmm14
 9559        vpxor	%xmm8, %xmm14, %xmm14
 9560        vpxor	%xmm14, %xmm6, %xmm6
 9561        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm6, %xmm6
 9562        vmovdqu	144(%rsp), %xmm0
 9563        vpxor	%xmm6, %xmm0, %xmm0
 9564        cmpl	$16, %r14d
 9565        je	L_AES_GCM_decrypt_avx1_cmp_tag_16
 9566        subq	$16, %rsp
 9567        xorq	%rcx, %rcx
 9568        xorq	%rbx, %rbx
 9569        vmovdqu	%xmm0, (%rsp)
 9570L_AES_GCM_decrypt_avx1_cmp_tag_loop:
 9571        movzbl	(%rsp,%rcx,1), %r13d
 9572        xorb	(%r8,%rcx,1), %r13b
 9573        orb	%r13b, %bl
 9574        incl	%ecx
 9575        cmpl	%r14d, %ecx
 9576        jne	L_AES_GCM_decrypt_avx1_cmp_tag_loop
 9577        cmpb	$0x00, %bl
 9578        sete	%bl
 9579        addq	$16, %rsp
 9580        xorq	%rcx, %rcx
 9581        jmp	L_AES_GCM_decrypt_avx1_cmp_tag_done
 9582L_AES_GCM_decrypt_avx1_cmp_tag_16:
 9583        vmovdqu	(%r8), %xmm1
 9584        vpcmpeqb	%xmm1, %xmm0, %xmm0
 9585        vpmovmskb	%xmm0, %rdx
 9586        # %%edx == 0xFFFF then return 1 else => return 0
 9587        xorl	%ebx, %ebx
 9588        cmpl	$0xffff, %edx
 9589        sete	%bl
 9590L_AES_GCM_decrypt_avx1_cmp_tag_done:
 9591        movl	%ebx, (%rbp)
 9592        vzeroupper
 9593        addq	$0xa8, %rsp
 9594        popq	%rbp
 9595        popq	%r15
 9596        popq	%r14
 9597        popq	%rbx
 9598        popq	%r12
 9599        popq	%r13
 9600        repz retq
 9601#ifndef __APPLE__
 9602.size	AES_GCM_decrypt_avx1,.-AES_GCM_decrypt_avx1
 9603#endif /* __APPLE__ */
 9604#ifdef WOLFSSL_AESGCM_STREAM
 9605#ifndef __APPLE__
 9606.text
 9607.globl	AES_GCM_init_avx1
 9608.type	AES_GCM_init_avx1,@function
 9609.align	16
 9610AES_GCM_init_avx1:
 9611#else
 9612.section	__TEXT,__text
 9613.globl	_AES_GCM_init_avx1
 9614.p2align	4
 9615_AES_GCM_init_avx1:
 9616#endif /* __APPLE__ */
 9617        pushq	%r12
 9618        pushq	%r13
 9619        movq	%rdx, %r10
 9620        movl	%ecx, %r11d
 9621        movq	24(%rsp), %rax
 9622        subq	$16, %rsp
 9623        vpxor	%xmm4, %xmm4, %xmm4
 9624        movl	%r11d, %edx
 9625        cmpl	$12, %edx
 9626        jne	L_AES_GCM_init_avx1_iv_not_12
 9627        # # Calculate values when IV is 12 bytes
 9628        # Set counter based on IV
 9629        movl	$0x1000000, %ecx
 9630        vmovq	(%r10), %xmm4
 9631        vpinsrd	$2, 8(%r10), %xmm4, %xmm4
 9632        vpinsrd	$3, %ecx, %xmm4, %xmm4
 9633        # H = Encrypt X(=0) and T = Encrypt counter
 9634        vmovdqa	(%rdi), %xmm5
 9635        vpxor	%xmm5, %xmm4, %xmm1
 9636        vmovdqa	16(%rdi), %xmm6
 9637        vaesenc	%xmm6, %xmm5, %xmm5
 9638        vaesenc	%xmm6, %xmm1, %xmm1
 9639        vmovdqa	32(%rdi), %xmm6
 9640        vaesenc	%xmm6, %xmm5, %xmm5
 9641        vaesenc	%xmm6, %xmm1, %xmm1
 9642        vmovdqa	48(%rdi), %xmm6
 9643        vaesenc	%xmm6, %xmm5, %xmm5
 9644        vaesenc	%xmm6, %xmm1, %xmm1
 9645        vmovdqa	64(%rdi), %xmm6
 9646        vaesenc	%xmm6, %xmm5, %xmm5
 9647        vaesenc	%xmm6, %xmm1, %xmm1
 9648        vmovdqa	80(%rdi), %xmm6
 9649        vaesenc	%xmm6, %xmm5, %xmm5
 9650        vaesenc	%xmm6, %xmm1, %xmm1
 9651        vmovdqa	96(%rdi), %xmm6
 9652        vaesenc	%xmm6, %xmm5, %xmm5
 9653        vaesenc	%xmm6, %xmm1, %xmm1
 9654        vmovdqa	112(%rdi), %xmm6
 9655        vaesenc	%xmm6, %xmm5, %xmm5
 9656        vaesenc	%xmm6, %xmm1, %xmm1
 9657        vmovdqa	128(%rdi), %xmm6
 9658        vaesenc	%xmm6, %xmm5, %xmm5
 9659        vaesenc	%xmm6, %xmm1, %xmm1
 9660        vmovdqa	144(%rdi), %xmm6
 9661        vaesenc	%xmm6, %xmm5, %xmm5
 9662        vaesenc	%xmm6, %xmm1, %xmm1
 9663        cmpl	$11, %esi
 9664        vmovdqa	160(%rdi), %xmm6
 9665        jl	L_AES_GCM_init_avx1_calc_iv_12_last
 9666        vaesenc	%xmm6, %xmm5, %xmm5
 9667        vaesenc	%xmm6, %xmm1, %xmm1
 9668        vmovdqa	176(%rdi), %xmm6
 9669        vaesenc	%xmm6, %xmm5, %xmm5
 9670        vaesenc	%xmm6, %xmm1, %xmm1
 9671        cmpl	$13, %esi
 9672        vmovdqa	192(%rdi), %xmm6
 9673        jl	L_AES_GCM_init_avx1_calc_iv_12_last
 9674        vaesenc	%xmm6, %xmm5, %xmm5
 9675        vaesenc	%xmm6, %xmm1, %xmm1
 9676        vmovdqa	208(%rdi), %xmm6
 9677        vaesenc	%xmm6, %xmm5, %xmm5
 9678        vaesenc	%xmm6, %xmm1, %xmm1
 9679        vmovdqa	224(%rdi), %xmm6
 9680L_AES_GCM_init_avx1_calc_iv_12_last:
 9681        vaesenclast	%xmm6, %xmm5, %xmm5
 9682        vaesenclast	%xmm6, %xmm1, %xmm1
 9683        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm5, %xmm5
 9684        vmovdqu	%xmm1, %xmm15
 9685        jmp	L_AES_GCM_init_avx1_iv_done
 9686L_AES_GCM_init_avx1_iv_not_12:
 9687        # Calculate values when IV is not 12 bytes
 9688        # H = Encrypt X(=0)
 9689        vmovdqa	(%rdi), %xmm5
 9690        vaesenc	16(%rdi), %xmm5, %xmm5
 9691        vaesenc	32(%rdi), %xmm5, %xmm5
 9692        vaesenc	48(%rdi), %xmm5, %xmm5
 9693        vaesenc	64(%rdi), %xmm5, %xmm5
 9694        vaesenc	80(%rdi), %xmm5, %xmm5
 9695        vaesenc	96(%rdi), %xmm5, %xmm5
 9696        vaesenc	112(%rdi), %xmm5, %xmm5
 9697        vaesenc	128(%rdi), %xmm5, %xmm5
 9698        vaesenc	144(%rdi), %xmm5, %xmm5
 9699        cmpl	$11, %esi
 9700        vmovdqa	160(%rdi), %xmm8
 9701        jl	L_AES_GCM_init_avx1_calc_iv_1_aesenc_avx_last
 9702        vaesenc	%xmm8, %xmm5, %xmm5
 9703        vaesenc	176(%rdi), %xmm5, %xmm5
 9704        cmpl	$13, %esi
 9705        vmovdqa	192(%rdi), %xmm8
 9706        jl	L_AES_GCM_init_avx1_calc_iv_1_aesenc_avx_last
 9707        vaesenc	%xmm8, %xmm5, %xmm5
 9708        vaesenc	208(%rdi), %xmm5, %xmm5
 9709        vmovdqa	224(%rdi), %xmm8
 9710L_AES_GCM_init_avx1_calc_iv_1_aesenc_avx_last:
 9711        vaesenclast	%xmm8, %xmm5, %xmm5
 9712        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm5, %xmm5
 9713        # Calc counter
 9714        # Initialization vector
 9715        cmpl	$0x00, %edx
 9716        movq	$0x00, %rcx
 9717        je	L_AES_GCM_init_avx1_calc_iv_done
 9718        cmpl	$16, %edx
 9719        jl	L_AES_GCM_init_avx1_calc_iv_lt16
 9720        andl	$0xfffffff0, %edx
 9721L_AES_GCM_init_avx1_calc_iv_16_loop:
 9722        vmovdqu	(%r10,%rcx,1), %xmm7
 9723        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm7, %xmm7
 9724        vpxor	%xmm7, %xmm4, %xmm4
 9725        # ghash_gfmul_avx
 9726        vpshufd	$0x4e, %xmm4, %xmm1
 9727        vpshufd	$0x4e, %xmm5, %xmm2
 9728        vpclmulqdq	$0x11, %xmm4, %xmm5, %xmm3
 9729        vpclmulqdq	$0x00, %xmm4, %xmm5, %xmm0
 9730        vpxor	%xmm4, %xmm1, %xmm1
 9731        vpxor	%xmm5, %xmm2, %xmm2
 9732        vpclmulqdq	$0x00, %xmm2, %xmm1, %xmm1
 9733        vpxor	%xmm0, %xmm1, %xmm1
 9734        vpxor	%xmm3, %xmm1, %xmm1
 9735        vmovdqa	%xmm0, %xmm6
 9736        vmovdqa	%xmm3, %xmm4
 9737        vpslldq	$8, %xmm1, %xmm2
 9738        vpsrldq	$8, %xmm1, %xmm1
 9739        vpxor	%xmm2, %xmm6, %xmm6
 9740        vpxor	%xmm1, %xmm4, %xmm4
 9741        vpsrld	$31, %xmm6, %xmm0
 9742        vpsrld	$31, %xmm4, %xmm1
 9743        vpslld	$0x01, %xmm6, %xmm6
 9744        vpslld	$0x01, %xmm4, %xmm4
 9745        vpsrldq	$12, %xmm0, %xmm2
 9746        vpslldq	$4, %xmm0, %xmm0
 9747        vpslldq	$4, %xmm1, %xmm1
 9748        vpor	%xmm2, %xmm4, %xmm4
 9749        vpor	%xmm0, %xmm6, %xmm6
 9750        vpor	%xmm1, %xmm4, %xmm4
 9751        vpslld	$31, %xmm6, %xmm0
 9752        vpslld	$30, %xmm6, %xmm1
 9753        vpslld	$25, %xmm6, %xmm2
 9754        vpxor	%xmm1, %xmm0, %xmm0
 9755        vpxor	%xmm2, %xmm0, %xmm0
 9756        vmovdqa	%xmm0, %xmm1
 9757        vpsrldq	$4, %xmm1, %xmm1
 9758        vpslldq	$12, %xmm0, %xmm0
 9759        vpxor	%xmm0, %xmm6, %xmm6
 9760        vpsrld	$0x01, %xmm6, %xmm2
 9761        vpsrld	$2, %xmm6, %xmm3
 9762        vpsrld	$7, %xmm6, %xmm0
 9763        vpxor	%xmm3, %xmm2, %xmm2
 9764        vpxor	%xmm0, %xmm2, %xmm2
 9765        vpxor	%xmm1, %xmm2, %xmm2
 9766        vpxor	%xmm6, %xmm2, %xmm2
 9767        vpxor	%xmm2, %xmm4, %xmm4
 9768        addl	$16, %ecx
 9769        cmpl	%edx, %ecx
 9770        jl	L_AES_GCM_init_avx1_calc_iv_16_loop
 9771        movl	%r11d, %edx
 9772        cmpl	%edx, %ecx
 9773        je	L_AES_GCM_init_avx1_calc_iv_done
 9774L_AES_GCM_init_avx1_calc_iv_lt16:
 9775        subq	$16, %rsp
 9776        vpxor	%xmm7, %xmm7, %xmm7
 9777        xorl	%r13d, %r13d
 9778        vmovdqu	%xmm7, (%rsp)
 9779L_AES_GCM_init_avx1_calc_iv_loop:
 9780        movzbl	(%r10,%rcx,1), %r12d
 9781        movb	%r12b, (%rsp,%r13,1)
 9782        incl	%ecx
 9783        incl	%r13d
 9784        cmpl	%edx, %ecx
 9785        jl	L_AES_GCM_init_avx1_calc_iv_loop
 9786        vmovdqu	(%rsp), %xmm7
 9787        addq	$16, %rsp
 9788        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm7, %xmm7
 9789        vpxor	%xmm7, %xmm4, %xmm4
 9790        # ghash_gfmul_avx
 9791        vpshufd	$0x4e, %xmm4, %xmm1
 9792        vpshufd	$0x4e, %xmm5, %xmm2
 9793        vpclmulqdq	$0x11, %xmm4, %xmm5, %xmm3
 9794        vpclmulqdq	$0x00, %xmm4, %xmm5, %xmm0
 9795        vpxor	%xmm4, %xmm1, %xmm1
 9796        vpxor	%xmm5, %xmm2, %xmm2
 9797        vpclmulqdq	$0x00, %xmm2, %xmm1, %xmm1
 9798        vpxor	%xmm0, %xmm1, %xmm1
 9799        vpxor	%xmm3, %xmm1, %xmm1
 9800        vmovdqa	%xmm0, %xmm6
 9801        vmovdqa	%xmm3, %xmm4
 9802        vpslldq	$8, %xmm1, %xmm2
 9803        vpsrldq	$8, %xmm1, %xmm1
 9804        vpxor	%xmm2, %xmm6, %xmm6
 9805        vpxor	%xmm1, %xmm4, %xmm4
 9806        vpsrld	$31, %xmm6, %xmm0
 9807        vpsrld	$31, %xmm4, %xmm1
 9808        vpslld	$0x01, %xmm6, %xmm6
 9809        vpslld	$0x01, %xmm4, %xmm4
 9810        vpsrldq	$12, %xmm0, %xmm2
 9811        vpslldq	$4, %xmm0, %xmm0
 9812        vpslldq	$4, %xmm1, %xmm1
 9813        vpor	%xmm2, %xmm4, %xmm4
 9814        vpor	%xmm0, %xmm6, %xmm6
 9815        vpor	%xmm1, %xmm4, %xmm4
 9816        vpslld	$31, %xmm6, %xmm0
 9817        vpslld	$30, %xmm6, %xmm1
 9818        vpslld	$25, %xmm6, %xmm2
 9819        vpxor	%xmm1, %xmm0, %xmm0
 9820        vpxor	%xmm2, %xmm0, %xmm0
 9821        vmovdqa	%xmm0, %xmm1
 9822        vpsrldq	$4, %xmm1, %xmm1
 9823        vpslldq	$12, %xmm0, %xmm0
 9824        vpxor	%xmm0, %xmm6, %xmm6
 9825        vpsrld	$0x01, %xmm6, %xmm2
 9826        vpsrld	$2, %xmm6, %xmm3
 9827        vpsrld	$7, %xmm6, %xmm0
 9828        vpxor	%xmm3, %xmm2, %xmm2
 9829        vpxor	%xmm0, %xmm2, %xmm2
 9830        vpxor	%xmm1, %xmm2, %xmm2
 9831        vpxor	%xmm6, %xmm2, %xmm2
 9832        vpxor	%xmm2, %xmm4, %xmm4
 9833L_AES_GCM_init_avx1_calc_iv_done:
 9834        # T = Encrypt counter
 9835        vpxor	%xmm0, %xmm0, %xmm0
 9836        shll	$3, %edx
 9837        vmovq	%rdx, %xmm0
 9838        vpxor	%xmm0, %xmm4, %xmm4
 9839        # ghash_gfmul_avx
 9840        vpshufd	$0x4e, %xmm4, %xmm1
 9841        vpshufd	$0x4e, %xmm5, %xmm2
 9842        vpclmulqdq	$0x11, %xmm4, %xmm5, %xmm3
 9843        vpclmulqdq	$0x00, %xmm4, %xmm5, %xmm0
 9844        vpxor	%xmm4, %xmm1, %xmm1
 9845        vpxor	%xmm5, %xmm2, %xmm2
 9846        vpclmulqdq	$0x00, %xmm2, %xmm1, %xmm1
 9847        vpxor	%xmm0, %xmm1, %xmm1
 9848        vpxor	%xmm3, %xmm1, %xmm1
 9849        vmovdqa	%xmm0, %xmm6
 9850        vmovdqa	%xmm3, %xmm4
 9851        vpslldq	$8, %xmm1, %xmm2
 9852        vpsrldq	$8, %xmm1, %xmm1
 9853        vpxor	%xmm2, %xmm6, %xmm6
 9854        vpxor	%xmm1, %xmm4, %xmm4
 9855        vpsrld	$31, %xmm6, %xmm0
 9856        vpsrld	$31, %xmm4, %xmm1
 9857        vpslld	$0x01, %xmm6, %xmm6
 9858        vpslld	$0x01, %xmm4, %xmm4
 9859        vpsrldq	$12, %xmm0, %xmm2
 9860        vpslldq	$4, %xmm0, %xmm0
 9861        vpslldq	$4, %xmm1, %xmm1
 9862        vpor	%xmm2, %xmm4, %xmm4
 9863        vpor	%xmm0, %xmm6, %xmm6
 9864        vpor	%xmm1, %xmm4, %xmm4
 9865        vpslld	$31, %xmm6, %xmm0
 9866        vpslld	$30, %xmm6, %xmm1
 9867        vpslld	$25, %xmm6, %xmm2
 9868        vpxor	%xmm1, %xmm0, %xmm0
 9869        vpxor	%xmm2, %xmm0, %xmm0
 9870        vmovdqa	%xmm0, %xmm1
 9871        vpsrldq	$4, %xmm1, %xmm1
 9872        vpslldq	$12, %xmm0, %xmm0
 9873        vpxor	%xmm0, %xmm6, %xmm6
 9874        vpsrld	$0x01, %xmm6, %xmm2
 9875        vpsrld	$2, %xmm6, %xmm3
 9876        vpsrld	$7, %xmm6, %xmm0
 9877        vpxor	%xmm3, %xmm2, %xmm2
 9878        vpxor	%xmm0, %xmm2, %xmm2
 9879        vpxor	%xmm1, %xmm2, %xmm2
 9880        vpxor	%xmm6, %xmm2, %xmm2
 9881        vpxor	%xmm2, %xmm4, %xmm4
 9882        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm4, %xmm4
 9883        #   Encrypt counter
 9884        vmovdqa	(%rdi), %xmm7
 9885        vpxor	%xmm4, %xmm7, %xmm7
 9886        vaesenc	16(%rdi), %xmm7, %xmm7
 9887        vaesenc	32(%rdi), %xmm7, %xmm7
 9888        vaesenc	48(%rdi), %xmm7, %xmm7
 9889        vaesenc	64(%rdi), %xmm7, %xmm7
 9890        vaesenc	80(%rdi), %xmm7, %xmm7
 9891        vaesenc	96(%rdi), %xmm7, %xmm7
 9892        vaesenc	112(%rdi), %xmm7, %xmm7
 9893        vaesenc	128(%rdi), %xmm7, %xmm7
 9894        vaesenc	144(%rdi), %xmm7, %xmm7
 9895        cmpl	$11, %esi
 9896        vmovdqa	160(%rdi), %xmm8
 9897        jl	L_AES_GCM_init_avx1_calc_iv_2_aesenc_avx_last
 9898        vaesenc	%xmm8, %xmm7, %xmm7
 9899        vaesenc	176(%rdi), %xmm7, %xmm7
 9900        cmpl	$13, %esi
 9901        vmovdqa	192(%rdi), %xmm8
 9902        jl	L_AES_GCM_init_avx1_calc_iv_2_aesenc_avx_last
 9903        vaesenc	%xmm8, %xmm7, %xmm7
 9904        vaesenc	208(%rdi), %xmm7, %xmm7
 9905        vmovdqa	224(%rdi), %xmm8
 9906L_AES_GCM_init_avx1_calc_iv_2_aesenc_avx_last:
 9907        vaesenclast	%xmm8, %xmm7, %xmm7
 9908        vmovdqu	%xmm7, %xmm15
 9909L_AES_GCM_init_avx1_iv_done:
 9910        vmovdqa	%xmm15, (%rax)
 9911        vpshufb	L_avx1_aes_gcm_bswap_epi64(%rip), %xmm4, %xmm4
 9912        vpaddd	L_avx1_aes_gcm_one(%rip), %xmm4, %xmm4
 9913        vmovdqa	%xmm5, (%r8)
 9914        vmovdqa	%xmm4, (%r9)
 9915        addq	$16, %rsp
 9916        popq	%r13
 9917        popq	%r12
 9918        repz retq
 9919#ifndef __APPLE__
 9920.size	AES_GCM_init_avx1,.-AES_GCM_init_avx1
 9921#endif /* __APPLE__ */
 9922#ifndef __APPLE__
 9923.text
 9924.globl	AES_GCM_aad_update_avx1
 9925.type	AES_GCM_aad_update_avx1,@function
 9926.align	16
 9927AES_GCM_aad_update_avx1:
 9928#else
 9929.section	__TEXT,__text
 9930.globl	_AES_GCM_aad_update_avx1
 9931.p2align	4
 9932_AES_GCM_aad_update_avx1:
 9933#endif /* __APPLE__ */
 9934        movq	%rcx, %rax
 9935        vmovdqa	(%rdx), %xmm5
 9936        vmovdqa	(%rax), %xmm6
 9937        xorl	%ecx, %ecx
 9938L_AES_GCM_aad_update_avx1_16_loop:
 9939        vmovdqu	(%rdi,%rcx,1), %xmm7
 9940        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm7, %xmm7
 9941        vpxor	%xmm7, %xmm5, %xmm5
 9942        # ghash_gfmul_avx
 9943        vpshufd	$0x4e, %xmm5, %xmm1
 9944        vpshufd	$0x4e, %xmm6, %xmm2
 9945        vpclmulqdq	$0x11, %xmm5, %xmm6, %xmm3
 9946        vpclmulqdq	$0x00, %xmm5, %xmm6, %xmm0
 9947        vpxor	%xmm5, %xmm1, %xmm1
 9948        vpxor	%xmm6, %xmm2, %xmm2
 9949        vpclmulqdq	$0x00, %xmm2, %xmm1, %xmm1
 9950        vpxor	%xmm0, %xmm1, %xmm1
 9951        vpxor	%xmm3, %xmm1, %xmm1
 9952        vmovdqa	%xmm0, %xmm4
 9953        vmovdqa	%xmm3, %xmm5
 9954        vpslldq	$8, %xmm1, %xmm2
 9955        vpsrldq	$8, %xmm1, %xmm1
 9956        vpxor	%xmm2, %xmm4, %xmm4
 9957        vpxor	%xmm1, %xmm5, %xmm5
 9958        vpsrld	$31, %xmm4, %xmm0
 9959        vpsrld	$31, %xmm5, %xmm1
 9960        vpslld	$0x01, %xmm4, %xmm4
 9961        vpslld	$0x01, %xmm5, %xmm5
 9962        vpsrldq	$12, %xmm0, %xmm2
 9963        vpslldq	$4, %xmm0, %xmm0
 9964        vpslldq	$4, %xmm1, %xmm1
 9965        vpor	%xmm2, %xmm5, %xmm5
 9966        vpor	%xmm0, %xmm4, %xmm4
 9967        vpor	%xmm1, %xmm5, %xmm5
 9968        vpslld	$31, %xmm4, %xmm0
 9969        vpslld	$30, %xmm4, %xmm1
 9970        vpslld	$25, %xmm4, %xmm2
 9971        vpxor	%xmm1, %xmm0, %xmm0
 9972        vpxor	%xmm2, %xmm0, %xmm0
 9973        vmovdqa	%xmm0, %xmm1
 9974        vpsrldq	$4, %xmm1, %xmm1
 9975        vpslldq	$12, %xmm0, %xmm0
 9976        vpxor	%xmm0, %xmm4, %xmm4
 9977        vpsrld	$0x01, %xmm4, %xmm2
 9978        vpsrld	$2, %xmm4, %xmm3
 9979        vpsrld	$7, %xmm4, %xmm0
 9980        vpxor	%xmm3, %xmm2, %xmm2
 9981        vpxor	%xmm0, %xmm2, %xmm2
 9982        vpxor	%xmm1, %xmm2, %xmm2
 9983        vpxor	%xmm4, %xmm2, %xmm2
 9984        vpxor	%xmm2, %xmm5, %xmm5
 9985        addl	$16, %ecx
 9986        cmpl	%esi, %ecx
 9987        jl	L_AES_GCM_aad_update_avx1_16_loop
 9988        vmovdqa	%xmm5, (%rdx)
 9989        repz retq
 9990#ifndef __APPLE__
 9991.size	AES_GCM_aad_update_avx1,.-AES_GCM_aad_update_avx1
 9992#endif /* __APPLE__ */
 9993#ifndef __APPLE__
 9994.text
 9995.globl	AES_GCM_encrypt_block_avx1
 9996.type	AES_GCM_encrypt_block_avx1,@function
 9997.align	16
 9998AES_GCM_encrypt_block_avx1:
 9999#else
10000.section	__TEXT,__text
10001.globl	_AES_GCM_encrypt_block_avx1
10002.p2align	4
10003_AES_GCM_encrypt_block_avx1:
10004#endif /* __APPLE__ */
10005        movq	%rdx, %r10
10006        movq	%rcx, %r11
10007        vmovdqu	(%r8), %xmm1
10008        vpshufb	L_avx1_aes_gcm_bswap_epi64(%rip), %xmm1, %xmm0
10009        vpaddd	L_avx1_aes_gcm_one(%rip), %xmm1, %xmm1
10010        vmovdqu	%xmm1, (%r8)
10011        vpxor	(%rdi), %xmm0, %xmm0
10012        vaesenc	16(%rdi), %xmm0, %xmm0
10013        vaesenc	32(%rdi), %xmm0, %xmm0
10014        vaesenc	48(%rdi), %xmm0, %xmm0
10015        vaesenc	64(%rdi), %xmm0, %xmm0
10016        vaesenc	80(%rdi), %xmm0, %xmm0
10017        vaesenc	96(%rdi), %xmm0, %xmm0
10018        vaesenc	112(%rdi), %xmm0, %xmm0
10019        vaesenc	128(%rdi), %xmm0, %xmm0
10020        vaesenc	144(%rdi), %xmm0, %xmm0
10021        cmpl	$11, %esi
10022        vmovdqa	160(%rdi), %xmm1
10023        jl	L_AES_GCM_encrypt_block_avx1_aesenc_block_last
10024        vaesenc	%xmm1, %xmm0, %xmm0
10025        vaesenc	176(%rdi), %xmm0, %xmm0
10026        cmpl	$13, %esi
10027        vmovdqa	192(%rdi), %xmm1
10028        jl	L_AES_GCM_encrypt_block_avx1_aesenc_block_last
10029        vaesenc	%xmm1, %xmm0, %xmm0
10030        vaesenc	208(%rdi), %xmm0, %xmm0
10031        vmovdqa	224(%rdi), %xmm1
10032L_AES_GCM_encrypt_block_avx1_aesenc_block_last:
10033        vaesenclast	%xmm1, %xmm0, %xmm0
10034        vmovdqu	(%r11), %xmm1
10035        vpxor	%xmm1, %xmm0, %xmm0
10036        vmovdqu	%xmm0, (%r10)
10037        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0
10038        vzeroupper
10039        repz retq
10040#ifndef __APPLE__
10041.size	AES_GCM_encrypt_block_avx1,.-AES_GCM_encrypt_block_avx1
10042#endif /* __APPLE__ */
10043#ifndef __APPLE__
10044.text
10045.globl	AES_GCM_ghash_block_avx1
10046.type	AES_GCM_ghash_block_avx1,@function
10047.align	16
10048AES_GCM_ghash_block_avx1:
10049#else
10050.section	__TEXT,__text
10051.globl	_AES_GCM_ghash_block_avx1
10052.p2align	4
10053_AES_GCM_ghash_block_avx1:
10054#endif /* __APPLE__ */
10055        vmovdqa	(%rsi), %xmm4
10056        vmovdqa	(%rdx), %xmm5
10057        vmovdqu	(%rdi), %xmm7
10058        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm7, %xmm7
10059        vpxor	%xmm7, %xmm4, %xmm4
10060        # ghash_gfmul_avx
10061        vpshufd	$0x4e, %xmm4, %xmm1
10062        vpshufd	$0x4e, %xmm5, %xmm2
10063        vpclmulqdq	$0x11, %xmm4, %xmm5, %xmm3
10064        vpclmulqdq	$0x00, %xmm4, %xmm5, %xmm0
10065        vpxor	%xmm4, %xmm1, %xmm1
10066        vpxor	%xmm5, %xmm2, %xmm2
10067        vpclmulqdq	$0x00, %xmm2, %xmm1, %xmm1
10068        vpxor	%xmm0, %xmm1, %xmm1
10069        vpxor	%xmm3, %xmm1, %xmm1
10070        vmovdqa	%xmm0, %xmm6
10071        vmovdqa	%xmm3, %xmm4
10072        vpslldq	$8, %xmm1, %xmm2
10073        vpsrldq	$8, %xmm1, %xmm1
10074        vpxor	%xmm2, %xmm6, %xmm6
10075        vpxor	%xmm1, %xmm4, %xmm4
10076        vpsrld	$31, %xmm6, %xmm0
10077        vpsrld	$31, %xmm4, %xmm1
10078        vpslld	$0x01, %xmm6, %xmm6
10079        vpslld	$0x01, %xmm4, %xmm4
10080        vpsrldq	$12, %xmm0, %xmm2
10081        vpslldq	$4, %xmm0, %xmm0
10082        vpslldq	$4, %xmm1, %xmm1
10083        vpor	%xmm2, %xmm4, %xmm4
10084        vpor	%xmm0, %xmm6, %xmm6
10085        vpor	%xmm1, %xmm4, %xmm4
10086        vpslld	$31, %xmm6, %xmm0
10087        vpslld	$30, %xmm6, %xmm1
10088        vpslld	$25, %xmm6, %xmm2
10089        vpxor	%xmm1, %xmm0, %xmm0
10090        vpxor	%xmm2, %xmm0, %xmm0
10091        vmovdqa	%xmm0, %xmm1
10092        vpsrldq	$4, %xmm1, %xmm1
10093        vpslldq	$12, %xmm0, %xmm0
10094        vpxor	%xmm0, %xmm6, %xmm6
10095        vpsrld	$0x01, %xmm6, %xmm2
10096        vpsrld	$2, %xmm6, %xmm3
10097        vpsrld	$7, %xmm6, %xmm0
10098        vpxor	%xmm3, %xmm2, %xmm2
10099        vpxor	%xmm0, %xmm2, %xmm2
10100        vpxor	%xmm1, %xmm2, %xmm2
10101        vpxor	%xmm6, %xmm2, %xmm2
10102        vpxor	%xmm2, %xmm4, %xmm4
10103        vmovdqa	%xmm4, (%rsi)
10104        vzeroupper
10105        repz retq
10106#ifndef __APPLE__
10107.size	AES_GCM_ghash_block_avx1,.-AES_GCM_ghash_block_avx1
10108#endif /* __APPLE__ */
10109#ifndef __APPLE__
10110.text
10111.globl	AES_GCM_encrypt_update_avx1
10112.type	AES_GCM_encrypt_update_avx1,@function
10113.align	16
10114AES_GCM_encrypt_update_avx1:
10115#else
10116.section	__TEXT,__text
10117.globl	_AES_GCM_encrypt_update_avx1
10118.p2align	4
10119_AES_GCM_encrypt_update_avx1:
10120#endif /* __APPLE__ */
10121        pushq	%r13
10122        pushq	%r12
10123        pushq	%r14
10124        movq	%rdx, %r10
10125        movq	%rcx, %r11
10126        movq	32(%rsp), %rax
10127        movq	40(%rsp), %r12
10128        subq	$0xa0, %rsp
10129        vmovdqa	(%r9), %xmm6
10130        vmovdqa	(%rax), %xmm5
10131        vpsrlq	$63, %xmm5, %xmm9
10132        vpsllq	$0x01, %xmm5, %xmm8
10133        vpslldq	$8, %xmm9, %xmm9
10134        vpor	%xmm9, %xmm8, %xmm8
10135        vpshufd	$0xff, %xmm5, %xmm5
10136        vpsrad	$31, %xmm5, %xmm5
10137        vpand	L_avx1_aes_gcm_mod2_128(%rip), %xmm5, %xmm5
10138        vpxor	%xmm8, %xmm5, %xmm5
10139        xorl	%r14d, %r14d
10140        cmpl	$0x80, %r8d
10141        movl	%r8d, %r13d
10142        jl	L_AES_GCM_encrypt_update_avx1_done_128
10143        andl	$0xffffff80, %r13d
10144        vmovdqa	%xmm6, %xmm2
10145        # H ^ 1
10146        vmovdqu	%xmm5, (%rsp)
10147        # H ^ 2
10148        vpclmulqdq	$0x00, %xmm5, %xmm5, %xmm8
10149        vpclmulqdq	$0x11, %xmm5, %xmm5, %xmm0
10150        vpslld	$31, %xmm8, %xmm12
10151        vpslld	$30, %xmm8, %xmm13
10152        vpslld	$25, %xmm8, %xmm14
10153        vpxor	%xmm13, %xmm12, %xmm12
10154        vpxor	%xmm14, %xmm12, %xmm12
10155        vpsrldq	$4, %xmm12, %xmm13
10156        vpslldq	$12, %xmm12, %xmm12
10157        vpxor	%xmm12, %xmm8, %xmm8
10158        vpsrld	$0x01, %xmm8, %xmm14
10159        vpsrld	$2, %xmm8, %xmm10
10160        vpsrld	$7, %xmm8, %xmm9
10161        vpxor	%xmm10, %xmm14, %xmm14
10162        vpxor	%xmm9, %xmm14, %xmm14
10163        vpxor	%xmm13, %xmm14, %xmm14
10164        vpxor	%xmm8, %xmm14, %xmm14
10165        vpxor	%xmm14, %xmm0, %xmm0
10166        vmovdqu	%xmm0, 16(%rsp)
10167        # H ^ 3
10168        # ghash_gfmul_red_avx
10169        vpshufd	$0x4e, %xmm5, %xmm9
10170        vpshufd	$0x4e, %xmm0, %xmm10
10171        vpclmulqdq	$0x11, %xmm5, %xmm0, %xmm11
10172        vpclmulqdq	$0x00, %xmm5, %xmm0, %xmm8
10173        vpxor	%xmm5, %xmm9, %xmm9
10174        vpxor	%xmm0, %xmm10, %xmm10
10175        vpclmulqdq	$0x00, %xmm10, %xmm9, %xmm9
10176        vpxor	%xmm8, %xmm9, %xmm9
10177        vpxor	%xmm11, %xmm9, %xmm9
10178        vpslldq	$8, %xmm9, %xmm10
10179        vpsrldq	$8, %xmm9, %xmm9
10180        vpxor	%xmm10, %xmm8, %xmm8
10181        vpxor	%xmm9, %xmm11, %xmm1
10182        vpslld	$31, %xmm8, %xmm12
10183        vpslld	$30, %xmm8, %xmm13
10184        vpslld	$25, %xmm8, %xmm14
10185        vpxor	%xmm13, %xmm12, %xmm12
10186        vpxor	%xmm14, %xmm12, %xmm12
10187        vpsrldq	$4, %xmm12, %xmm13
10188        vpslldq	$12, %xmm12, %xmm12
10189        vpxor	%xmm12, %xmm8, %xmm8
10190        vpsrld	$0x01, %xmm8, %xmm14
10191        vpsrld	$2, %xmm8, %xmm10
10192        vpsrld	$7, %xmm8, %xmm9
10193        vpxor	%xmm10, %xmm14, %xmm14
10194        vpxor	%xmm9, %xmm14, %xmm14
10195        vpxor	%xmm13, %xmm14, %xmm14
10196        vpxor	%xmm8, %xmm14, %xmm14
10197        vpxor	%xmm14, %xmm1, %xmm1
10198        vmovdqu	%xmm1, 32(%rsp)
10199        # H ^ 4
10200        vpclmulqdq	$0x00, %xmm0, %xmm0, %xmm8
10201        vpclmulqdq	$0x11, %xmm0, %xmm0, %xmm3
10202        vpslld	$31, %xmm8, %xmm12
10203        vpslld	$30, %xmm8, %xmm13
10204        vpslld	$25, %xmm8, %xmm14
10205        vpxor	%xmm13, %xmm12, %xmm12
10206        vpxor	%xmm14, %xmm12, %xmm12
10207        vpsrldq	$4, %xmm12, %xmm13
10208        vpslldq	$12, %xmm12, %xmm12
10209        vpxor	%xmm12, %xmm8, %xmm8
10210        vpsrld	$0x01, %xmm8, %xmm14
10211        vpsrld	$2, %xmm8, %xmm10
10212        vpsrld	$7, %xmm8, %xmm9
10213        vpxor	%xmm10, %xmm14, %xmm14
10214        vpxor	%xmm9, %xmm14, %xmm14
10215        vpxor	%xmm13, %xmm14, %xmm14
10216        vpxor	%xmm8, %xmm14, %xmm14
10217        vpxor	%xmm14, %xmm3, %xmm3
10218        vmovdqu	%xmm3, 48(%rsp)
10219        # H ^ 5
10220        # ghash_gfmul_red_avx
10221        vpshufd	$0x4e, %xmm0, %xmm9
10222        vpshufd	$0x4e, %xmm1, %xmm10
10223        vpclmulqdq	$0x11, %xmm0, %xmm1, %xmm11
10224        vpclmulqdq	$0x00, %xmm0, %xmm1, %xmm8
10225        vpxor	%xmm0, %xmm9, %xmm9
10226        vpxor	%xmm1, %xmm10, %xmm10
10227        vpclmulqdq	$0x00, %xmm10, %xmm9, %xmm9
10228        vpxor	%xmm8, %xmm9, %xmm9
10229        vpxor	%xmm11, %xmm9, %xmm9
10230        vpslldq	$8, %xmm9, %xmm10
10231        vpsrldq	$8, %xmm9, %xmm9
10232        vpxor	%xmm10, %xmm8, %xmm8
10233        vpxor	%xmm9, %xmm11, %xmm7
10234        vpslld	$31, %xmm8, %xmm12
10235        vpslld	$30, %xmm8, %xmm13
10236        vpslld	$25, %xmm8, %xmm14
10237        vpxor	%xmm13, %xmm12, %xmm12
10238        vpxor	%xmm14, %xmm12, %xmm12
10239        vpsrldq	$4, %xmm12, %xmm13
10240        vpslldq	$12, %xmm12, %xmm12
10241        vpxor	%xmm12, %xmm8, %xmm8
10242        vpsrld	$0x01, %xmm8, %xmm14
10243        vpsrld	$2, %xmm8, %xmm10
10244        vpsrld	$7, %xmm8, %xmm9
10245        vpxor	%xmm10, %xmm14, %xmm14
10246        vpxor	%xmm9, %xmm14, %xmm14
10247        vpxor	%xmm13, %xmm14, %xmm14
10248        vpxor	%xmm8, %xmm14, %xmm14
10249        vpxor	%xmm14, %xmm7, %xmm7
10250        vmovdqu	%xmm7, 64(%rsp)
10251        # H ^ 6
10252        vpclmulqdq	$0x00, %xmm1, %xmm1, %xmm8
10253        vpclmulqdq	$0x11, %xmm1, %xmm1, %xmm7
10254        vpslld	$31, %xmm8, %xmm12
10255        vpslld	$30, %xmm8, %xmm13
10256        vpslld	$25, %xmm8, %xmm14
10257        vpxor	%xmm13, %xmm12, %xmm12
10258        vpxor	%xmm14, %xmm12, %xmm12
10259        vpsrldq	$4, %xmm12, %xmm13
10260        vpslldq	$12, %xmm12, %xmm12
10261        vpxor	%xmm12, %xmm8, %xmm8
10262        vpsrld	$0x01, %xmm8, %xmm14
10263        vpsrld	$2, %xmm8, %xmm10
10264        vpsrld	$7, %xmm8, %xmm9
10265        vpxor	%xmm10, %xmm14, %xmm14
10266        vpxor	%xmm9, %xmm14, %xmm14
10267        vpxor	%xmm13, %xmm14, %xmm14
10268        vpxor	%xmm8, %xmm14, %xmm14
10269        vpxor	%xmm14, %xmm7, %xmm7
10270        vmovdqu	%xmm7, 80(%rsp)
10271        # H ^ 7
10272        # ghash_gfmul_red_avx
10273        vpshufd	$0x4e, %xmm1, %xmm9
10274        vpshufd	$0x4e, %xmm3, %xmm10
10275        vpclmulqdq	$0x11, %xmm1, %xmm3, %xmm11
10276        vpclmulqdq	$0x00, %xmm1, %xmm3, %xmm8
10277        vpxor	%xmm1, %xmm9, %xmm9
10278        vpxor	%xmm3, %xmm10, %xmm10
10279        vpclmulqdq	$0x00, %xmm10, %xmm9, %xmm9
10280        vpxor	%xmm8, %xmm9, %xmm9
10281        vpxor	%xmm11, %xmm9, %xmm9
10282        vpslldq	$8, %xmm9, %xmm10
10283        vpsrldq	$8, %xmm9, %xmm9
10284        vpxor	%xmm10, %xmm8, %xmm8
10285        vpxor	%xmm9, %xmm11, %xmm7
10286        vpslld	$31, %xmm8, %xmm12
10287        vpslld	$30, %xmm8, %xmm13
10288        vpslld	$25, %xmm8, %xmm14
10289        vpxor	%xmm13, %xmm12, %xmm12
10290        vpxor	%xmm14, %xmm12, %xmm12
10291        vpsrldq	$4, %xmm12, %xmm13
10292        vpslldq	$12, %xmm12, %xmm12
10293        vpxor	%xmm12, %xmm8, %xmm8
10294        vpsrld	$0x01, %xmm8, %xmm14
10295        vpsrld	$2, %xmm8, %xmm10
10296        vpsrld	$7, %xmm8, %xmm9
10297        vpxor	%xmm10, %xmm14, %xmm14
10298        vpxor	%xmm9, %xmm14, %xmm14
10299        vpxor	%xmm13, %xmm14, %xmm14
10300        vpxor	%xmm8, %xmm14, %xmm14
10301        vpxor	%xmm14, %xmm7, %xmm7
10302        vmovdqu	%xmm7, 96(%rsp)
10303        # H ^ 8
10304        vpclmulqdq	$0x00, %xmm3, %xmm3, %xmm8
10305        vpclmulqdq	$0x11, %xmm3, %xmm3, %xmm7
10306        vpslld	$31, %xmm8, %xmm12
10307        vpslld	$30, %xmm8, %xmm13
10308        vpslld	$25, %xmm8, %xmm14
10309        vpxor	%xmm13, %xmm12, %xmm12
10310        vpxor	%xmm14, %xmm12, %xmm12
10311        vpsrldq	$4, %xmm12, %xmm13
10312        vpslldq	$12, %xmm12, %xmm12
10313        vpxor	%xmm12, %xmm8, %xmm8
10314        vpsrld	$0x01, %xmm8, %xmm14
10315        vpsrld	$2, %xmm8, %xmm10
10316        vpsrld	$7, %xmm8, %xmm9
10317        vpxor	%xmm10, %xmm14, %xmm14
10318        vpxor	%xmm9, %xmm14, %xmm14
10319        vpxor	%xmm13, %xmm14, %xmm14
10320        vpxor	%xmm8, %xmm14, %xmm14
10321        vpxor	%xmm14, %xmm7, %xmm7
10322        vmovdqu	%xmm7, 112(%rsp)
10323        # First 128 bytes of input
10324        vmovdqu	(%r12), %xmm0
10325        vmovdqa	L_avx1_aes_gcm_bswap_epi64(%rip), %xmm1
10326        vpshufb	%xmm1, %xmm0, %xmm8
10327        vpaddd	L_avx1_aes_gcm_one(%rip), %xmm0, %xmm9
10328        vpshufb	%xmm1, %xmm9, %xmm9
10329        vpaddd	L_avx1_aes_gcm_two(%rip), %xmm0, %xmm10
10330        vpshufb	%xmm1, %xmm10, %xmm10
10331        vpaddd	L_avx1_aes_gcm_three(%rip), %xmm0, %xmm11
10332        vpshufb	%xmm1, %xmm11, %xmm11
10333        vpaddd	L_avx1_aes_gcm_four(%rip), %xmm0, %xmm12
10334        vpshufb	%xmm1, %xmm12, %xmm12
10335        vpaddd	L_avx1_aes_gcm_five(%rip), %xmm0, %xmm13
10336        vpshufb	%xmm1, %xmm13, %xmm13
10337        vpaddd	L_avx1_aes_gcm_six(%rip), %xmm0, %xmm14
10338        vpshufb	%xmm1, %xmm14, %xmm14
10339        vpaddd	L_avx1_aes_gcm_seven(%rip), %xmm0, %xmm15
10340        vpshufb	%xmm1, %xmm15, %xmm15
10341        vpaddd	L_avx1_aes_gcm_eight(%rip), %xmm0, %xmm0
10342        vmovdqa	(%rdi), %xmm7
10343        vmovdqu	%xmm0, (%r12)
10344        vpxor	%xmm7, %xmm8, %xmm8
10345        vpxor	%xmm7, %xmm9, %xmm9
10346        vpxor	%xmm7, %xmm10, %xmm10
10347        vpxor	%xmm7, %xmm11, %xmm11
10348        vpxor	%xmm7, %xmm12, %xmm12
10349        vpxor	%xmm7, %xmm13, %xmm13
10350        vpxor	%xmm7, %xmm14, %xmm14
10351        vpxor	%xmm7, %xmm15, %xmm15
10352        vmovdqa	16(%rdi), %xmm7
10353        vaesenc	%xmm7, %xmm8, %xmm8
10354        vaesenc	%xmm7, %xmm9, %xmm9
10355        vaesenc	%xmm7, %xmm10, %xmm10
10356        vaesenc	%xmm7, %xmm11, %xmm11
10357        vaesenc	%xmm7, %xmm12, %xmm12
10358        vaesenc	%xmm7, %xmm13, %xmm13
10359        vaesenc	%xmm7, %xmm14, %xmm14
10360        vaesenc	%xmm7, %xmm15, %xmm15
10361        vmovdqa	32(%rdi), %xmm7
10362        vaesenc	%xmm7, %xmm8, %xmm8
10363        vaesenc	%xmm7, %xmm9, %xmm9
10364        vaesenc	%xmm7, %xmm10, %xmm10
10365        vaesenc	%xmm7, %xmm11, %xmm11
10366        vaesenc	%xmm7, %xmm12, %xmm12
10367        vaesenc	%xmm7, %xmm13, %xmm13
10368        vaesenc	%xmm7, %xmm14, %xmm14
10369        vaesenc	%xmm7, %xmm15, %xmm15
10370        vmovdqa	48(%rdi), %xmm7
10371        vaesenc	%xmm7, %xmm8, %xmm8
10372        vaesenc	%xmm7, %xmm9, %xmm9
10373        vaesenc	%xmm7, %xmm10, %xmm10
10374        vaesenc	%xmm7, %xmm11, %xmm11
10375        vaesenc	%xmm7, %xmm12, %xmm12
10376        vaesenc	%xmm7, %xmm13, %xmm13
10377        vaesenc	%xmm7, %xmm14, %xmm14
10378        vaesenc	%xmm7, %xmm15, %xmm15
10379        vmovdqa	64(%rdi), %xmm7
10380        vaesenc	%xmm7, %xmm8, %xmm8
10381        vaesenc	%xmm7, %xmm9, %xmm9
10382        vaesenc	%xmm7, %xmm10, %xmm10
10383        vaesenc	%xmm7, %xmm11, %xmm11
10384        vaesenc	%xmm7, %xmm12, %xmm12
10385        vaesenc	%xmm7, %xmm13, %xmm13
10386        vaesenc	%xmm7, %xmm14, %xmm14
10387        vaesenc	%xmm7, %xmm15, %xmm15
10388        vmovdqa	80(%rdi), %xmm7
10389        vaesenc	%xmm7, %xmm8, %xmm8
10390        vaesenc	%xmm7, %xmm9, %xmm9
10391        vaesenc	%xmm7, %xmm10, %xmm10
10392        vaesenc	%xmm7, %xmm11, %xmm11
10393        vaesenc	%xmm7, %xmm12, %xmm12
10394        vaesenc	%xmm7, %xmm13, %xmm13
10395        vaesenc	%xmm7, %xmm14, %xmm14
10396        vaesenc	%xmm7, %xmm15, %xmm15
10397        vmovdqa	96(%rdi), %xmm7
10398        vaesenc	%xmm7, %xmm8, %xmm8
10399        vaesenc	%xmm7, %xmm9, %xmm9
10400        vaesenc	%xmm7, %xmm10, %xmm10
10401        vaesenc	%xmm7, %xmm11, %xmm11
10402        vaesenc	%xmm7, %xmm12, %xmm12
10403        vaesenc	%xmm7, %xmm13, %xmm13
10404        vaesenc	%xmm7, %xmm14, %xmm14
10405        vaesenc	%xmm7, %xmm15, %xmm15
10406        vmovdqa	112(%rdi), %xmm7
10407        vaesenc	%xmm7, %xmm8, %xmm8
10408        vaesenc	%xmm7, %xmm9, %xmm9
10409        vaesenc	%xmm7, %xmm10, %xmm10
10410        vaesenc	%xmm7, %xmm11, %xmm11
10411        vaesenc	%xmm7, %xmm12, %xmm12
10412        vaesenc	%xmm7, %xmm13, %xmm13
10413        vaesenc	%xmm7, %xmm14, %xmm14
10414        vaesenc	%xmm7, %xmm15, %xmm15
10415        vmovdqa	128(%rdi), %xmm7
10416        vaesenc	%xmm7, %xmm8, %xmm8
10417        vaesenc	%xmm7, %xmm9, %xmm9
10418        vaesenc	%xmm7, %xmm10, %xmm10
10419        vaesenc	%xmm7, %xmm11, %xmm11
10420        vaesenc	%xmm7, %xmm12, %xmm12
10421        vaesenc	%xmm7, %xmm13, %xmm13
10422        vaesenc	%xmm7, %xmm14, %xmm14
10423        vaesenc	%xmm7, %xmm15, %xmm15
10424        vmovdqa	144(%rdi), %xmm7
10425        vaesenc	%xmm7, %xmm8, %xmm8
10426        vaesenc	%xmm7, %xmm9, %xmm9
10427        vaesenc	%xmm7, %xmm10, %xmm10
10428        vaesenc	%xmm7, %xmm11, %xmm11
10429        vaesenc	%xmm7, %xmm12, %xmm12
10430        vaesenc	%xmm7, %xmm13, %xmm13
10431        vaesenc	%xmm7, %xmm14, %xmm14
10432        vaesenc	%xmm7, %xmm15, %xmm15
10433        cmpl	$11, %esi
10434        vmovdqa	160(%rdi), %xmm7
10435        jl	L_AES_GCM_encrypt_update_avx1_aesenc_128_enc_done
10436        vaesenc	%xmm7, %xmm8, %xmm8
10437        vaesenc	%xmm7, %xmm9, %xmm9
10438        vaesenc	%xmm7, %xmm10, %xmm10
10439        vaesenc	%xmm7, %xmm11, %xmm11
10440        vaesenc	%xmm7, %xmm12, %xmm12
10441        vaesenc	%xmm7, %xmm13, %xmm13
10442        vaesenc	%xmm7, %xmm14, %xmm14
10443        vaesenc	%xmm7, %xmm15, %xmm15
10444        vmovdqa	176(%rdi), %xmm7
10445        vaesenc	%xmm7, %xmm8, %xmm8
10446        vaesenc	%xmm7, %xmm9, %xmm9
10447        vaesenc	%xmm7, %xmm10, %xmm10
10448        vaesenc	%xmm7, %xmm11, %xmm11
10449        vaesenc	%xmm7, %xmm12, %xmm12
10450        vaesenc	%xmm7, %xmm13, %xmm13
10451        vaesenc	%xmm7, %xmm14, %xmm14
10452        vaesenc	%xmm7, %xmm15, %xmm15
10453        cmpl	$13, %esi
10454        vmovdqa	192(%rdi), %xmm7
10455        jl	L_AES_GCM_encrypt_update_avx1_aesenc_128_enc_done
10456        vaesenc	%xmm7, %xmm8, %xmm8
10457        vaesenc	%xmm7, %xmm9, %xmm9
10458        vaesenc	%xmm7, %xmm10, %xmm10
10459        vaesenc	%xmm7, %xmm11, %xmm11
10460        vaesenc	%xmm7, %xmm12, %xmm12
10461        vaesenc	%xmm7, %xmm13, %xmm13
10462        vaesenc	%xmm7, %xmm14, %xmm14
10463        vaesenc	%xmm7, %xmm15, %xmm15
10464        vmovdqa	208(%rdi), %xmm7
10465        vaesenc	%xmm7, %xmm8, %xmm8
10466        vaesenc	%xmm7, %xmm9, %xmm9
10467        vaesenc	%xmm7, %xmm10, %xmm10
10468        vaesenc	%xmm7, %xmm11, %xmm11
10469        vaesenc	%xmm7, %xmm12, %xmm12
10470        vaesenc	%xmm7, %xmm13, %xmm13
10471        vaesenc	%xmm7, %xmm14, %xmm14
10472        vaesenc	%xmm7, %xmm15, %xmm15
10473        vmovdqa	224(%rdi), %xmm7
10474L_AES_GCM_encrypt_update_avx1_aesenc_128_enc_done:
10475        vaesenclast	%xmm7, %xmm8, %xmm8
10476        vaesenclast	%xmm7, %xmm9, %xmm9
10477        vmovdqu	(%r11), %xmm0
10478        vmovdqu	16(%r11), %xmm1
10479        vpxor	%xmm0, %xmm8, %xmm8
10480        vpxor	%xmm1, %xmm9, %xmm9
10481        vmovdqu	%xmm8, (%r10)
10482        vmovdqu	%xmm9, 16(%r10)
10483        vaesenclast	%xmm7, %xmm10, %xmm10
10484        vaesenclast	%xmm7, %xmm11, %xmm11
10485        vmovdqu	32(%r11), %xmm0
10486        vmovdqu	48(%r11), %xmm1
10487        vpxor	%xmm0, %xmm10, %xmm10
10488        vpxor	%xmm1, %xmm11, %xmm11
10489        vmovdqu	%xmm10, 32(%r10)
10490        vmovdqu	%xmm11, 48(%r10)
10491        vaesenclast	%xmm7, %xmm12, %xmm12
10492        vaesenclast	%xmm7, %xmm13, %xmm13
10493        vmovdqu	64(%r11), %xmm0
10494        vmovdqu	80(%r11), %xmm1
10495        vpxor	%xmm0, %xmm12, %xmm12
10496        vpxor	%xmm1, %xmm13, %xmm13
10497        vmovdqu	%xmm12, 64(%r10)
10498        vmovdqu	%xmm13, 80(%r10)
10499        vaesenclast	%xmm7, %xmm14, %xmm14
10500        vaesenclast	%xmm7, %xmm15, %xmm15
10501        vmovdqu	96(%r11), %xmm0
10502        vmovdqu	112(%r11), %xmm1
10503        vpxor	%xmm0, %xmm14, %xmm14
10504        vpxor	%xmm1, %xmm15, %xmm15
10505        vmovdqu	%xmm14, 96(%r10)
10506        vmovdqu	%xmm15, 112(%r10)
10507        cmpl	$0x80, %r13d
10508        movl	$0x80, %r14d
10509        jle	L_AES_GCM_encrypt_update_avx1_end_128
10510        # More 128 bytes of input
10511L_AES_GCM_encrypt_update_avx1_ghash_128:
10512        leaq	(%r11,%r14,1), %rcx
10513        leaq	(%r10,%r14,1), %rdx
10514        vmovdqu	(%r12), %xmm0
10515        vmovdqa	L_avx1_aes_gcm_bswap_epi64(%rip), %xmm1
10516        vpshufb	%xmm1, %xmm0, %xmm8
10517        vpaddd	L_avx1_aes_gcm_one(%rip), %xmm0, %xmm9
10518        vpshufb	%xmm1, %xmm9, %xmm9
10519        vpaddd	L_avx1_aes_gcm_two(%rip), %xmm0, %xmm10
10520        vpshufb	%xmm1, %xmm10, %xmm10
10521        vpaddd	L_avx1_aes_gcm_three(%rip), %xmm0, %xmm11
10522        vpshufb	%xmm1, %xmm11, %xmm11
10523        vpaddd	L_avx1_aes_gcm_four(%rip), %xmm0, %xmm12
10524        vpshufb	%xmm1, %xmm12, %xmm12
10525        vpaddd	L_avx1_aes_gcm_five(%rip), %xmm0, %xmm13
10526        vpshufb	%xmm1, %xmm13, %xmm13
10527        vpaddd	L_avx1_aes_gcm_six(%rip), %xmm0, %xmm14
10528        vpshufb	%xmm1, %xmm14, %xmm14
10529        vpaddd	L_avx1_aes_gcm_seven(%rip), %xmm0, %xmm15
10530        vpshufb	%xmm1, %xmm15, %xmm15
10531        vpaddd	L_avx1_aes_gcm_eight(%rip), %xmm0, %xmm0
10532        vmovdqa	(%rdi), %xmm7
10533        vmovdqu	%xmm0, (%r12)
10534        vpxor	%xmm7, %xmm8, %xmm8
10535        vpxor	%xmm7, %xmm9, %xmm9
10536        vpxor	%xmm7, %xmm10, %xmm10
10537        vpxor	%xmm7, %xmm11, %xmm11
10538        vpxor	%xmm7, %xmm12, %xmm12
10539        vpxor	%xmm7, %xmm13, %xmm13
10540        vpxor	%xmm7, %xmm14, %xmm14
10541        vpxor	%xmm7, %xmm15, %xmm15
10542        vmovdqu	112(%rsp), %xmm7
10543        vmovdqu	-128(%rdx), %xmm0
10544        vaesenc	16(%rdi), %xmm8, %xmm8
10545        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0
10546        vpxor	%xmm2, %xmm0, %xmm0
10547        vpshufd	$0x4e, %xmm7, %xmm1
10548        vpshufd	$0x4e, %xmm0, %xmm5
10549        vpxor	%xmm7, %xmm1, %xmm1
10550        vpxor	%xmm0, %xmm5, %xmm5
10551        vpclmulqdq	$0x11, %xmm7, %xmm0, %xmm3
10552        vaesenc	16(%rdi), %xmm9, %xmm9
10553        vaesenc	16(%rdi), %xmm10, %xmm10
10554        vpclmulqdq	$0x00, %xmm7, %xmm0, %xmm2
10555        vaesenc	16(%rdi), %xmm11, %xmm11
10556        vaesenc	16(%rdi), %xmm12, %xmm12
10557        vpclmulqdq	$0x00, %xmm5, %xmm1, %xmm1
10558        vaesenc	16(%rdi), %xmm13, %xmm13
10559        vaesenc	16(%rdi), %xmm14, %xmm14
10560        vaesenc	16(%rdi), %xmm15, %xmm15
10561        vpxor	%xmm2, %xmm1, %xmm1
10562        vpxor	%xmm3, %xmm1, %xmm1
10563        vmovdqu	96(%rsp), %xmm7
10564        vmovdqu	-112(%rdx), %xmm0
10565        vpshufd	$0x4e, %xmm7, %xmm4
10566        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0
10567        vaesenc	32(%rdi), %xmm8, %xmm8
10568        vpxor	%xmm7, %xmm4, %xmm4
10569        vpshufd	$0x4e, %xmm0, %xmm5
10570        vpxor	%xmm0, %xmm5, %xmm5
10571        vpclmulqdq	$0x11, %xmm7, %xmm0, %xmm6
10572        vaesenc	32(%rdi), %xmm9, %xmm9
10573        vaesenc	32(%rdi), %xmm10, %xmm10
10574        vpclmulqdq	$0x00, %xmm7, %xmm0, %xmm7
10575        vaesenc	32(%rdi), %xmm11, %xmm11
10576        vaesenc	32(%rdi), %xmm12, %xmm12
10577        vpclmulqdq	$0x00, %xmm5, %xmm4, %xmm4
10578        vaesenc	32(%rdi), %xmm13, %xmm13
10579        vaesenc	32(%rdi), %xmm14, %xmm14
10580        vaesenc	32(%rdi), %xmm15, %xmm15
10581        vpxor	%xmm7, %xmm1, %xmm1
10582        vpxor	%xmm7, %xmm2, %xmm2
10583        vpxor	%xmm6, %xmm1, %xmm1
10584        vpxor	%xmm6, %xmm3, %xmm3
10585        vpxor	%xmm4, %xmm1, %xmm1
10586        vmovdqu	80(%rsp), %xmm7
10587        vmovdqu	-96(%rdx), %xmm0
10588        vpshufd	$0x4e, %xmm7, %xmm4
10589        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0
10590        vaesenc	48(%rdi), %xmm8, %xmm8
10591        vpxor	%xmm7, %xmm4, %xmm4
10592        vpshufd	$0x4e, %xmm0, %xmm5
10593        vpxor	%xmm0, %xmm5, %xmm5
10594        vpclmulqdq	$0x11, %xmm7, %xmm0, %xmm6
10595        vaesenc	48(%rdi), %xmm9, %xmm9
10596        vaesenc	48(%rdi), %xmm10, %xmm10
10597        vpclmulqdq	$0x00, %xmm7, %xmm0, %xmm7
10598        vaesenc	48(%rdi), %xmm11, %xmm11
10599        vaesenc	48(%rdi), %xmm12, %xmm12
10600        vpclmulqdq	$0x00, %xmm5, %xmm4, %xmm4
10601        vaesenc	48(%rdi), %xmm13, %xmm13
10602        vaesenc	48(%rdi), %xmm14, %xmm14
10603        vaesenc	48(%rdi), %xmm15, %xmm15
10604        vpxor	%xmm7, %xmm1, %xmm1
10605        vpxor	%xmm7, %xmm2, %xmm2
10606        vpxor	%xmm6, %xmm1, %xmm1
10607        vpxor	%xmm6, %xmm3, %xmm3
10608        vpxor	%xmm4, %xmm1, %xmm1
10609        vmovdqu	64(%rsp), %xmm7
10610        vmovdqu	-80(%rdx), %xmm0
10611        vpshufd	$0x4e, %xmm7, %xmm4
10612        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0
10613        vaesenc	64(%rdi), %xmm8, %xmm8
10614        vpxor	%xmm7, %xmm4, %xmm4
10615        vpshufd	$0x4e, %xmm0, %xmm5
10616        vpxor	%xmm0, %xmm5, %xmm5
10617        vpclmulqdq	$0x11, %xmm7, %xmm0, %xmm6
10618        vaesenc	64(%rdi), %xmm9, %xmm9
10619        vaesenc	64(%rdi), %xmm10, %xmm10
10620        vpclmulqdq	$0x00, %xmm7, %xmm0, %xmm7
10621        vaesenc	64(%rdi), %xmm11, %xmm11
10622        vaesenc	64(%rdi), %xmm12, %xmm12
10623        vpclmulqdq	$0x00, %xmm5, %xmm4, %xmm4
10624        vaesenc	64(%rdi), %xmm13, %xmm13
10625        vaesenc	64(%rdi), %xmm14, %xmm14
10626        vaesenc	64(%rdi), %xmm15, %xmm15
10627        vpxor	%xmm7, %xmm1, %xmm1
10628        vpxor	%xmm7, %xmm2, %xmm2
10629        vpxor	%xmm6, %xmm1, %xmm1
10630        vpxor	%xmm6, %xmm3, %xmm3
10631        vpxor	%xmm4, %xmm1, %xmm1
10632        vmovdqu	48(%rsp), %xmm7
10633        vmovdqu	-64(%rdx), %xmm0
10634        vpshufd	$0x4e, %xmm7, %xmm4
10635        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0
10636        vaesenc	80(%rdi), %xmm8, %xmm8
10637        vpxor	%xmm7, %xmm4, %xmm4
10638        vpshufd	$0x4e, %xmm0, %xmm5
10639        vpxor	%xmm0, %xmm5, %xmm5
10640        vpclmulqdq	$0x11, %xmm7, %xmm0, %xmm6
10641        vaesenc	80(%rdi), %xmm9, %xmm9
10642        vaesenc	80(%rdi), %xmm10, %xmm10
10643        vpclmulqdq	$0x00, %xmm7, %xmm0, %xmm7
10644        vaesenc	80(%rdi), %xmm11, %xmm11
10645        vaesenc	80(%rdi), %xmm12, %xmm12
10646        vpclmulqdq	$0x00, %xmm5, %xmm4, %xmm4
10647        vaesenc	80(%rdi), %xmm13, %xmm13
10648        vaesenc	80(%rdi), %xmm14, %xmm14
10649        vaesenc	80(%rdi), %xmm15, %xmm15
10650        vpxor	%xmm7, %xmm1, %xmm1
10651        vpxor	%xmm7, %xmm2, %xmm2
10652        vpxor	%xmm6, %xmm1, %xmm1
10653        vpxor	%xmm6, %xmm3, %xmm3
10654        vpxor	%xmm4, %xmm1, %xmm1
10655        vmovdqu	32(%rsp), %xmm7
10656        vmovdqu	-48(%rdx), %xmm0
10657        vpshufd	$0x4e, %xmm7, %xmm4
10658        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0
10659        vaesenc	96(%rdi), %xmm8, %xmm8
10660        vpxor	%xmm7, %xmm4, %xmm4
10661        vpshufd	$0x4e, %xmm0, %xmm5
10662        vpxor	%xmm0, %xmm5, %xmm5
10663        vpclmulqdq	$0x11, %xmm7, %xmm0, %xmm6
10664        vaesenc	96(%rdi), %xmm9, %xmm9
10665        vaesenc	96(%rdi), %xmm10, %xmm10
10666        vpclmulqdq	$0x00, %xmm7, %xmm0, %xmm7
10667        vaesenc	96(%rdi), %xmm11, %xmm11
10668        vaesenc	96(%rdi), %xmm12, %xmm12
10669        vpclmulqdq	$0x00, %xmm5, %xmm4, %xmm4
10670        vaesenc	96(%rdi), %xmm13, %xmm13
10671        vaesenc	96(%rdi), %xmm14, %xmm14
10672        vaesenc	96(%rdi), %xmm15, %xmm15
10673        vpxor	%xmm7, %xmm1, %xmm1
10674        vpxor	%xmm7, %xmm2, %xmm2
10675        vpxor	%xmm6, %xmm1, %xmm1
10676        vpxor	%xmm6, %xmm3, %xmm3
10677        vpxor	%xmm4, %xmm1, %xmm1
10678        vmovdqu	16(%rsp), %xmm7
10679        vmovdqu	-32(%rdx), %xmm0
10680        vpshufd	$0x4e, %xmm7, %xmm4
10681        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0
10682        vaesenc	112(%rdi), %xmm8, %xmm8
10683        vpxor	%xmm7, %xmm4, %xmm4
10684        vpshufd	$0x4e, %xmm0, %xmm5
10685        vpxor	%xmm0, %xmm5, %xmm5
10686        vpclmulqdq	$0x11, %xmm7, %xmm0, %xmm6
10687        vaesenc	112(%rdi), %xmm9, %xmm9
10688        vaesenc	112(%rdi), %xmm10, %xmm10
10689        vpclmulqdq	$0x00, %xmm7, %xmm0, %xmm7
10690        vaesenc	112(%rdi), %xmm11, %xmm11
10691        vaesenc	112(%rdi), %xmm12, %xmm12
10692        vpclmulqdq	$0x00, %xmm5, %xmm4, %xmm4
10693        vaesenc	112(%rdi), %xmm13, %xmm13
10694        vaesenc	112(%rdi), %xmm14, %xmm14
10695        vaesenc	112(%rdi), %xmm15, %xmm15
10696        vpxor	%xmm7, %xmm1, %xmm1
10697        vpxor	%xmm7, %xmm2, %xmm2
10698        vpxor	%xmm6, %xmm1, %xmm1
10699        vpxor	%xmm6, %xmm3, %xmm3
10700        vpxor	%xmm4, %xmm1, %xmm1
10701        vmovdqu	(%rsp), %xmm7
10702        vmovdqu	-16(%rdx), %xmm0
10703        vpshufd	$0x4e, %xmm7, %xmm4
10704        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0
10705        vaesenc	128(%rdi), %xmm8, %xmm8
10706        vpxor	%xmm7, %xmm4, %xmm4
10707        vpshufd	$0x4e, %xmm0, %xmm5
10708        vpxor	%xmm0, %xmm5, %xmm5
10709        vpclmulqdq	$0x11, %xmm7, %xmm0, %xmm6
10710        vaesenc	128(%rdi), %xmm9, %xmm9
10711        vaesenc	128(%rdi), %xmm10, %xmm10
10712        vpclmulqdq	$0x00, %xmm7, %xmm0, %xmm7
10713        vaesenc	128(%rdi), %xmm11, %xmm11
10714        vaesenc	128(%rdi), %xmm12, %xmm12
10715        vpclmulqdq	$0x00, %xmm5, %xmm4, %xmm4
10716        vaesenc	128(%rdi), %xmm13, %xmm13
10717        vaesenc	128(%rdi), %xmm14, %xmm14
10718        vaesenc	128(%rdi), %xmm15, %xmm15
10719        vpxor	%xmm7, %xmm1, %xmm1
10720        vpxor	%xmm7, %xmm2, %xmm2
10721        vpxor	%xmm6, %xmm1, %xmm1
10722        vpxor	%xmm6, %xmm3, %xmm3
10723        vpxor	%xmm4, %xmm1, %xmm1
10724        vpslldq	$8, %xmm1, %xmm5
10725        vpsrldq	$8, %xmm1, %xmm1
10726        vaesenc	144(%rdi), %xmm8, %xmm8
10727        vpxor	%xmm5, %xmm2, %xmm2
10728        vpxor	%xmm1, %xmm3, %xmm3
10729        vaesenc	144(%rdi), %xmm9, %xmm9
10730        vpslld	$31, %xmm2, %xmm7
10731        vpslld	$30, %xmm2, %xmm4
10732        vpslld	$25, %xmm2, %xmm5
10733        vaesenc	144(%rdi), %xmm10, %xmm10
10734        vpxor	%xmm4, %xmm7, %xmm7
10735        vpxor	%xmm5, %xmm7, %xmm7
10736        vaesenc	144(%rdi), %xmm11, %xmm11
10737        vpsrldq	$4, %xmm7, %xmm4
10738        vpslldq	$12, %xmm7, %xmm7
10739        vaesenc	144(%rdi), %xmm12, %xmm12
10740        vpxor	%xmm7, %xmm2, %xmm2
10741        vpsrld	$0x01, %xmm2, %xmm5
10742        vaesenc	144(%rdi), %xmm13, %xmm13
10743        vpsrld	$2, %xmm2, %xmm1
10744        vpsrld	$7, %xmm2, %xmm0
10745        vaesenc	144(%rdi), %xmm14, %xmm14
10746        vpxor	%xmm1, %xmm5, %xmm5
10747        vpxor	%xmm0, %xmm5, %xmm5
10748        vaesenc	144(%rdi), %xmm15, %xmm15
10749        vpxor	%xmm4, %xmm5, %xmm5
10750        vpxor	%xmm5, %xmm2, %xmm2
10751        vpxor	%xmm3, %xmm2, %xmm2
10752        cmpl	$11, %esi
10753        vmovdqa	160(%rdi), %xmm7
10754        jl	L_AES_GCM_encrypt_update_avx1_aesenc_128_ghash_avx_done
10755        vaesenc	%xmm7, %xmm8, %xmm8
10756        vaesenc	%xmm7, %xmm9, %xmm9
10757        vaesenc	%xmm7, %xmm10, %xmm10
10758        vaesenc	%xmm7, %xmm11, %xmm11
10759        vaesenc	%xmm7, %xmm12, %xmm12
10760        vaesenc	%xmm7, %xmm13, %xmm13
10761        vaesenc	%xmm7, %xmm14, %xmm14
10762        vaesenc	%xmm7, %xmm15, %xmm15
10763        vmovdqa	176(%rdi), %xmm7
10764        vaesenc	%xmm7, %xmm8, %xmm8
10765        vaesenc	%xmm7, %xmm9, %xmm9
10766        vaesenc	%xmm7, %xmm10, %xmm10
10767        vaesenc	%xmm7, %xmm11, %xmm11
10768        vaesenc	%xmm7, %xmm12, %xmm12
10769        vaesenc	%xmm7, %xmm13, %xmm13
10770        vaesenc	%xmm7, %xmm14, %xmm14
10771        vaesenc	%xmm7, %xmm15, %xmm15
10772        cmpl	$13, %esi
10773        vmovdqa	192(%rdi), %xmm7
10774        jl	L_AES_GCM_encrypt_update_avx1_aesenc_128_ghash_avx_done
10775        vaesenc	%xmm7, %xmm8, %xmm8
10776        vaesenc	%xmm7, %xmm9, %xmm9
10777        vaesenc	%xmm7, %xmm10, %xmm10
10778        vaesenc	%xmm7, %xmm11, %xmm11
10779        vaesenc	%xmm7, %xmm12, %xmm12
10780        vaesenc	%xmm7, %xmm13, %xmm13
10781        vaesenc	%xmm7, %xmm14, %xmm14
10782        vaesenc	%xmm7, %xmm15, %xmm15
10783        vmovdqa	208(%rdi), %xmm7
10784        vaesenc	%xmm7, %xmm8, %xmm8
10785        vaesenc	%xmm7, %xmm9, %xmm9
10786        vaesenc	%xmm7, %xmm10, %xmm10
10787        vaesenc	%xmm7, %xmm11, %xmm11
10788        vaesenc	%xmm7, %xmm12, %xmm12
10789        vaesenc	%xmm7, %xmm13, %xmm13
10790        vaesenc	%xmm7, %xmm14, %xmm14
10791        vaesenc	%xmm7, %xmm15, %xmm15
10792        vmovdqa	224(%rdi), %xmm7
10793L_AES_GCM_encrypt_update_avx1_aesenc_128_ghash_avx_done:
10794        vaesenclast	%xmm7, %xmm8, %xmm8
10795        vaesenclast	%xmm7, %xmm9, %xmm9
10796        vmovdqu	(%rcx), %xmm0
10797        vmovdqu	16(%rcx), %xmm1
10798        vpxor	%xmm0, %xmm8, %xmm8
10799        vpxor	%xmm1, %xmm9, %xmm9
10800        vmovdqu	%xmm8, (%rdx)
10801        vmovdqu	%xmm9, 16(%rdx)
10802        vaesenclast	%xmm7, %xmm10, %xmm10
10803        vaesenclast	%xmm7, %xmm11, %xmm11
10804        vmovdqu	32(%rcx), %xmm0
10805        vmovdqu	48(%rcx), %xmm1
10806        vpxor	%xmm0, %xmm10, %xmm10
10807        vpxor	%xmm1, %xmm11, %xmm11
10808        vmovdqu	%xmm10, 32(%rdx)
10809        vmovdqu	%xmm11, 48(%rdx)
10810        vaesenclast	%xmm7, %xmm12, %xmm12
10811        vaesenclast	%xmm7, %xmm13, %xmm13
10812        vmovdqu	64(%rcx), %xmm0
10813        vmovdqu	80(%rcx), %xmm1
10814        vpxor	%xmm0, %xmm12, %xmm12
10815        vpxor	%xmm1, %xmm13, %xmm13
10816        vmovdqu	%xmm12, 64(%rdx)
10817        vmovdqu	%xmm13, 80(%rdx)
10818        vaesenclast	%xmm7, %xmm14, %xmm14
10819        vaesenclast	%xmm7, %xmm15, %xmm15
10820        vmovdqu	96(%rcx), %xmm0
10821        vmovdqu	112(%rcx), %xmm1
10822        vpxor	%xmm0, %xmm14, %xmm14
10823        vpxor	%xmm1, %xmm15, %xmm15
10824        vmovdqu	%xmm14, 96(%rdx)
10825        vmovdqu	%xmm15, 112(%rdx)
10826        addl	$0x80, %r14d
10827        cmpl	%r13d, %r14d
10828        jl	L_AES_GCM_encrypt_update_avx1_ghash_128
10829L_AES_GCM_encrypt_update_avx1_end_128:
10830        vmovdqa	L_avx1_aes_gcm_bswap_mask(%rip), %xmm4
10831        vpshufb	%xmm4, %xmm8, %xmm8
10832        vpshufb	%xmm4, %xmm9, %xmm9
10833        vpshufb	%xmm4, %xmm10, %xmm10
10834        vpshufb	%xmm4, %xmm11, %xmm11
10835        vpxor	%xmm2, %xmm8, %xmm8
10836        vpshufb	%xmm4, %xmm12, %xmm12
10837        vpshufb	%xmm4, %xmm13, %xmm13
10838        vpshufb	%xmm4, %xmm14, %xmm14
10839        vpshufb	%xmm4, %xmm15, %xmm15
10840        vmovdqu	(%rsp), %xmm7
10841        vmovdqu	16(%rsp), %xmm5
10842        # ghash_gfmul_avx
10843        vpshufd	$0x4e, %xmm15, %xmm1
10844        vpshufd	$0x4e, %xmm7, %xmm2
10845        vpclmulqdq	$0x11, %xmm15, %xmm7, %xmm3
10846        vpclmulqdq	$0x00, %xmm15, %xmm7, %xmm0
10847        vpxor	%xmm15, %xmm1, %xmm1
10848        vpxor	%xmm7, %xmm2, %xmm2
10849        vpclmulqdq	$0x00, %xmm2, %xmm1, %xmm1
10850        vpxor	%xmm0, %xmm1, %xmm1
10851        vpxor	%xmm3, %xmm1, %xmm1
10852        vmovdqa	%xmm0, %xmm4
10853        vmovdqa	%xmm3, %xmm6
10854        vpslldq	$8, %xmm1, %xmm2
10855        vpsrldq	$8, %xmm1, %xmm1
10856        vpxor	%xmm2, %xmm4, %xmm4
10857        vpxor	%xmm1, %xmm6, %xmm6
10858        # ghash_gfmul_xor_avx
10859        vpshufd	$0x4e, %xmm14, %xmm1
10860        vpshufd	$0x4e, %xmm5, %xmm2
10861        vpclmulqdq	$0x11, %xmm14, %xmm5, %xmm3
10862        vpclmulqdq	$0x00, %xmm14, %xmm5, %xmm0
10863        vpxor	%xmm14, %xmm1, %xmm1
10864        vpxor	%xmm5, %xmm2, %xmm2
10865        vpclmulqdq	$0x00, %xmm2, %xmm1, %xmm1
10866        vpxor	%xmm0, %xmm1, %xmm1
10867        vpxor	%xmm3, %xmm1, %xmm1
10868        vpxor	%xmm0, %xmm4, %xmm4
10869        vpxor	%xmm3, %xmm6, %xmm6
10870        vpslldq	$8, %xmm1, %xmm2
10871        vpsrldq	$8, %xmm1, %xmm1
10872        vpxor	%xmm2, %xmm4, %xmm4
10873        vpxor	%xmm1, %xmm6, %xmm6
10874        vmovdqu	32(%rsp), %xmm7
10875        vmovdqu	48(%rsp), %xmm5
10876        # ghash_gfmul_xor_avx
10877        vpshufd	$0x4e, %xmm13, %xmm1
10878        vpshufd	$0x4e, %xmm7, %xmm2
10879        vpclmulqdq	$0x11, %xmm13, %xmm7, %xmm3
10880        vpclmulqdq	$0x00, %xmm13, %xmm7, %xmm0
10881        vpxor	%xmm13, %xmm1, %xmm1
10882        vpxor	%xmm7, %xmm2, %xmm2
10883        vpclmulqdq	$0x00, %xmm2, %xmm1, %xmm1
10884        vpxor	%xmm0, %xmm1, %xmm1
10885        vpxor	%xmm3, %xmm1, %xmm1
10886        vpxor	%xmm0, %xmm4, %xmm4
10887        vpxor	%xmm3, %xmm6, %xmm6
10888        vpslldq	$8, %xmm1, %xmm2
10889        vpsrldq	$8, %xmm1, %xmm1
10890        vpxor	%xmm2, %xmm4, %xmm4
10891        vpxor	%xmm1, %xmm6, %xmm6
10892        # ghash_gfmul_xor_avx
10893        vpshufd	$0x4e, %xmm12, %xmm1
10894        vpshufd	$0x4e, %xmm5, %xmm2
10895        vpclmulqdq	$0x11, %xmm12, %xmm5, %xmm3
10896        vpclmulqdq	$0x00, %xmm12, %xmm5, %xmm0
10897        vpxor	%xmm12, %xmm1, %xmm1
10898        vpxor	%xmm5, %xmm2, %xmm2
10899        vpclmulqdq	$0x00, %xmm2, %xmm1, %xmm1
10900        vpxor	%xmm0, %xmm1, %xmm1
10901        vpxor	%xmm3, %xmm1, %xmm1
10902        vpxor	%xmm0, %xmm4, %xmm4
10903        vpxor	%xmm3, %xmm6, %xmm6
10904        vpslldq	$8, %xmm1, %xmm2
10905        vpsrldq	$8, %xmm1, %xmm1
10906        vpxor	%xmm2, %xmm4, %xmm4
10907        vpxor	%xmm1, %xmm6, %xmm6
10908        vmovdqu	64(%rsp), %xmm7
10909        vmovdqu	80(%rsp), %xmm5
10910        # ghash_gfmul_xor_avx
10911        vpshufd	$0x4e, %xmm11, %xmm1
10912        vpshufd	$0x4e, %xmm7, %xmm2
10913        vpclmulqdq	$0x11, %xmm11, %xmm7, %xmm3
10914        vpclmulqdq	$0x00, %xmm11, %xmm7, %xmm0
10915        vpxor	%xmm11, %xmm1, %xmm1
10916        vpxor	%xmm7, %xmm2, %xmm2
10917        vpclmulqdq	$0x00, %xmm2, %xmm1, %xmm1
10918        vpxor	%xmm0, %xmm1, %xmm1
10919        vpxor	%xmm3, %xmm1, %xmm1
10920        vpxor	%xmm0, %xmm4, %xmm4
10921        vpxor	%xmm3, %xmm6, %xmm6
10922        vpslldq	$8, %xmm1, %xmm2
10923        vpsrldq	$8, %xmm1, %xmm1
10924        vpxor	%xmm2, %xmm4, %xmm4
10925        vpxor	%xmm1, %xmm6, %xmm6
10926        # ghash_gfmul_xor_avx
10927        vpshufd	$0x4e, %xmm10, %xmm1
10928        vpshufd	$0x4e, %xmm5, %xmm2
10929        vpclmulqdq	$0x11, %xmm10, %xmm5, %xmm3
10930        vpclmulqdq	$0x00, %xmm10, %xmm5, %xmm0
10931        vpxor	%xmm10, %xmm1, %xmm1
10932        vpxor	%xmm5, %xmm2, %xmm2
10933        vpclmulqdq	$0x00, %xmm2, %xmm1, %xmm1
10934        vpxor	%xmm0, %xmm1, %xmm1
10935        vpxor	%xmm3, %xmm1, %xmm1
10936        vpxor	%xmm0, %xmm4, %xmm4
10937        vpxor	%xmm3, %xmm6, %xmm6
10938        vpslldq	$8, %xmm1, %xmm2
10939        vpsrldq	$8, %xmm1, %xmm1
10940        vpxor	%xmm2, %xmm4, %xmm4
10941        vpxor	%xmm1, %xmm6, %xmm6
10942        vmovdqu	96(%rsp), %xmm7
10943        vmovdqu	112(%rsp), %xmm5
10944        # ghash_gfmul_xor_avx
10945        vpshufd	$0x4e, %xmm9, %xmm1
10946        vpshufd	$0x4e, %xmm7, %xmm2
10947        vpclmulqdq	$0x11, %xmm9, %xmm7, %xmm3
10948        vpclmulqdq	$0x00, %xmm9, %xmm7, %xmm0
10949        vpxor	%xmm9, %xmm1, %xmm1
10950        vpxor	%xmm7, %xmm2, %xmm2
10951        vpclmulqdq	$0x00, %xmm2, %xmm1, %xmm1
10952        vpxor	%xmm0, %xmm1, %xmm1
10953        vpxor	%xmm3, %xmm1, %xmm1
10954        vpxor	%xmm0, %xmm4, %xmm4
10955        vpxor	%xmm3, %xmm6, %xmm6
10956        vpslldq	$8, %xmm1, %xmm2
10957        vpsrldq	$8, %xmm1, %xmm1
10958        vpxor	%xmm2, %xmm4, %xmm4
10959        vpxor	%xmm1, %xmm6, %xmm6
10960        # ghash_gfmul_xor_avx
10961        vpshufd	$0x4e, %xmm8, %xmm1
10962        vpshufd	$0x4e, %xmm5, %xmm2
10963        vpclmulqdq	$0x11, %xmm8, %xmm5, %xmm3
10964        vpclmulqdq	$0x00, %xmm8, %xmm5, %xmm0
10965        vpxor	%xmm8, %xmm1, %xmm1
10966        vpxor	%xmm5, %xmm2, %xmm2
10967        vpclmulqdq	$0x00, %xmm2, %xmm1, %xmm1
10968        vpxor	%xmm0, %xmm1, %xmm1
10969        vpxor	%xmm3, %xmm1, %xmm1
10970        vpxor	%xmm0, %xmm4, %xmm4
10971        vpxor	%xmm3, %xmm6, %xmm6
10972        vpslldq	$8, %xmm1, %xmm2
10973        vpsrldq	$8, %xmm1, %xmm1
10974        vpxor	%xmm2, %xmm4, %xmm4
10975        vpxor	%xmm1, %xmm6, %xmm6
10976        vpslld	$31, %xmm4, %xmm0
10977        vpslld	$30, %xmm4, %xmm1
10978        vpslld	$25, %xmm4, %xmm2
10979        vpxor	%xmm1, %xmm0, %xmm0
10980        vpxor	%xmm2, %xmm0, %xmm0
10981        vmovdqa	%xmm0, %xmm1
10982        vpsrldq	$4, %xmm1, %xmm1
10983        vpslldq	$12, %xmm0, %xmm0
10984        vpxor	%xmm0, %xmm4, %xmm4
10985        vpsrld	$0x01, %xmm4, %xmm2
10986        vpsrld	$2, %xmm4, %xmm3
10987        vpsrld	$7, %xmm4, %xmm0
10988        vpxor	%xmm3, %xmm2, %xmm2
10989        vpxor	%xmm0, %xmm2, %xmm2
10990        vpxor	%xmm1, %xmm2, %xmm2
10991        vpxor	%xmm4, %xmm2, %xmm2
10992        vpxor	%xmm2, %xmm6, %xmm6
10993        vmovdqu	(%rsp), %xmm5
10994L_AES_GCM_encrypt_update_avx1_done_128:
10995        movl	%r8d, %edx
10996        cmpl	%edx, %r14d
10997        jge	L_AES_GCM_encrypt_update_avx1_done_enc
10998        movl	%r8d, %r13d
10999        andl	$0xfffffff0, %r13d
11000        cmpl	%r13d, %r14d
11001        jge	L_AES_GCM_encrypt_update_avx1_last_block_done
11002        vmovdqu	(%r12), %xmm9
11003        vpshufb	L_avx1_aes_gcm_bswap_epi64(%rip), %xmm9, %xmm8
11004        vpaddd	L_avx1_aes_gcm_one(%rip), %xmm9, %xmm9
11005        vmovdqu	%xmm9, (%r12)
11006        vpxor	(%rdi), %xmm8, %xmm8
11007        vaesenc	16(%rdi), %xmm8, %xmm8
11008        vaesenc	32(%rdi), %xmm8, %xmm8
11009        vaesenc	48(%rdi), %xmm8, %xmm8
11010        vaesenc	64(%rdi), %xmm8, %xmm8
11011        vaesenc	80(%rdi), %xmm8, %xmm8
11012        vaesenc	96(%rdi), %xmm8, %xmm8
11013        vaesenc	112(%rdi), %xmm8, %xmm8
11014        vaesenc	128(%rdi), %xmm8, %xmm8
11015        vaesenc	144(%rdi), %xmm8, %xmm8
11016        cmpl	$11, %esi
11017        vmovdqa	160(%rdi), %xmm9
11018        jl	L_AES_GCM_encrypt_update_avx1_aesenc_block_last
11019        vaesenc	%xmm9, %xmm8, %xmm8
11020        vaesenc	176(%rdi), %xmm8, %xmm8
11021        cmpl	$13, %esi
11022        vmovdqa	192(%rdi), %xmm9
11023        jl	L_AES_GCM_encrypt_update_avx1_aesenc_block_last
11024        vaesenc	%xmm9, %xmm8, %xmm8
11025        vaesenc	208(%rdi), %xmm8, %xmm8
11026        vmovdqa	224(%rdi), %xmm9
11027L_AES_GCM_encrypt_update_avx1_aesenc_block_last:
11028        vaesenclast	%xmm9, %xmm8, %xmm8
11029        vmovdqu	(%r11,%r14,1), %xmm9
11030        vpxor	%xmm9, %xmm8, %xmm8
11031        vmovdqu	%xmm8, (%r10,%r14,1)
11032        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm8, %xmm8
11033        vpxor	%xmm8, %xmm6, %xmm6
11034        addl	$16, %r14d
11035        cmpl	%r13d, %r14d
11036        jge	L_AES_GCM_encrypt_update_avx1_last_block_ghash
11037L_AES_GCM_encrypt_update_avx1_last_block_start:
11038        vmovdqu	(%r11,%r14,1), %xmm13
11039        vmovdqu	(%r12), %xmm9
11040        vpshufb	L_avx1_aes_gcm_bswap_epi64(%rip), %xmm9, %xmm8
11041        vpaddd	L_avx1_aes_gcm_one(%rip), %xmm9, %xmm9
11042        vmovdqu	%xmm9, (%r12)
11043        vpxor	(%rdi), %xmm8, %xmm8
11044        vpclmulqdq	$16, %xmm5, %xmm6, %xmm10
11045        vaesenc	16(%rdi), %xmm8, %xmm8
11046        vaesenc	32(%rdi), %xmm8, %xmm8
11047        vpclmulqdq	$0x01, %xmm5, %xmm6, %xmm11
11048        vaesenc	48(%rdi), %xmm8, %xmm8
11049        vaesenc	64(%rdi), %xmm8, %xmm8
11050        vpclmulqdq	$0x00, %xmm5, %xmm6, %xmm12
11051        vaesenc	80(%rdi), %xmm8, %xmm8
11052        vpclmulqdq	$0x11, %xmm5, %xmm6, %xmm1
11053        vaesenc	96(%rdi), %xmm8, %xmm8
11054        vpxor	%xmm11, %xmm10, %xmm10
11055        vpslldq	$8, %xmm10, %xmm2
11056        vpsrldq	$8, %xmm10, %xmm10
11057        vaesenc	112(%rdi), %xmm8, %xmm8
11058        vpxor	%xmm12, %xmm2, %xmm2
11059        vpxor	%xmm10, %xmm1, %xmm3
11060        vmovdqa	L_avx1_aes_gcm_mod2_128(%rip), %xmm0
11061        vpclmulqdq	$16, %xmm0, %xmm2, %xmm11
11062        vaesenc	128(%rdi), %xmm8, %xmm8
11063        vpshufd	$0x4e, %xmm2, %xmm10
11064        vpxor	%xmm11, %xmm10, %xmm10
11065        vpclmulqdq	$16, %xmm0, %xmm10, %xmm11
11066        vaesenc	144(%rdi), %xmm8, %xmm8
11067        vpshufd	$0x4e, %xmm10, %xmm10
11068        vpxor	%xmm11, %xmm10, %xmm10
11069        vpxor	%xmm3, %xmm10, %xmm6
11070        cmpl	$11, %esi
11071        vmovdqa	160(%rdi), %xmm9
11072        jl	L_AES_GCM_encrypt_update_avx1_aesenc_gfmul_last
11073        vaesenc	%xmm9, %xmm8, %xmm8
11074        vaesenc	176(%rdi), %xmm8, %xmm8
11075        cmpl	$13, %esi
11076        vmovdqa	192(%rdi), %xmm9
11077        jl	L_AES_GCM_encrypt_update_avx1_aesenc_gfmul_last
11078        vaesenc	%xmm9, %xmm8, %xmm8
11079        vaesenc	208(%rdi), %xmm8, %xmm8
11080        vmovdqa	224(%rdi), %xmm9
11081L_AES_GCM_encrypt_update_avx1_aesenc_gfmul_last:
11082        vaesenclast	%xmm9, %xmm8, %xmm8
11083        vmovdqa	%xmm13, %xmm0
11084        vpxor	%xmm0, %xmm8, %xmm8
11085        vmovdqu	%xmm8, (%r10,%r14,1)
11086        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm8, %xmm8
11087        addl	$16, %r14d
11088        vpxor	%xmm8, %xmm6, %xmm6
11089        cmpl	%r13d, %r14d
11090        jl	L_AES_GCM_encrypt_update_avx1_last_block_start
11091L_AES_GCM_encrypt_update_avx1_last_block_ghash:
11092        # ghash_gfmul_red_avx
11093        vpshufd	$0x4e, %xmm5, %xmm9
11094        vpshufd	$0x4e, %xmm6, %xmm10
11095        vpclmulqdq	$0x11, %xmm5, %xmm6, %xmm11
11096        vpclmulqdq	$0x00, %xmm5, %xmm6, %xmm8
11097        vpxor	%xmm5, %xmm9, %xmm9
11098        vpxor	%xmm6, %xmm10, %xmm10
11099        vpclmulqdq	$0x00, %xmm10, %xmm9, %xmm9
11100        vpxor	%xmm8, %xmm9, %xmm9
11101        vpxor	%xmm11, %xmm9, %xmm9
11102        vpslldq	$8, %xmm9, %xmm10
11103        vpsrldq	$8, %xmm9, %xmm9
11104        vpxor	%xmm10, %xmm8, %xmm8
11105        vpxor	%xmm9, %xmm11, %xmm6
11106        vpslld	$31, %xmm8, %xmm12
11107        vpslld	$30, %xmm8, %xmm13
11108        vpslld	$25, %xmm8, %xmm14
11109        vpxor	%xmm13, %xmm12, %xmm12
11110        vpxor	%xmm14, %xmm12, %xmm12
11111        vpsrldq	$4, %xmm12, %xmm13
11112        vpslldq	$12, %xmm12, %xmm12
11113        vpxor	%xmm12, %xmm8, %xmm8
11114        vpsrld	$0x01, %xmm8, %xmm14
11115        vpsrld	$2, %xmm8, %xmm10
11116        vpsrld	$7, %xmm8, %xmm9
11117        vpxor	%xmm10, %xmm14, %xmm14
11118        vpxor	%xmm9, %xmm14, %xmm14
11119        vpxor	%xmm13, %xmm14, %xmm14
11120        vpxor	%xmm8, %xmm14, %xmm14
11121        vpxor	%xmm14, %xmm6, %xmm6
11122L_AES_GCM_encrypt_update_avx1_last_block_done:
11123L_AES_GCM_encrypt_update_avx1_done_enc:
11124        vmovdqa	%xmm6, (%r9)
11125        vzeroupper
11126        addq	$0xa0, %rsp
11127        popq	%r14
11128        popq	%r12
11129        popq	%r13
11130        repz retq
11131#ifndef __APPLE__
11132.size	AES_GCM_encrypt_update_avx1,.-AES_GCM_encrypt_update_avx1
11133#endif /* __APPLE__ */
11134#ifndef __APPLE__
11135.text
11136.globl	AES_GCM_encrypt_final_avx1
11137.type	AES_GCM_encrypt_final_avx1,@function
11138.align	16
11139AES_GCM_encrypt_final_avx1:
11140#else
11141.section	__TEXT,__text
11142.globl	_AES_GCM_encrypt_final_avx1
11143.p2align	4
11144_AES_GCM_encrypt_final_avx1:
11145#endif /* __APPLE__ */
11146        pushq	%r13
11147        movl	%edx, %eax
11148        movl	%ecx, %r10d
11149        movl	%r8d, %r11d
11150        movq	16(%rsp), %r8
11151        subq	$16, %rsp
11152        vmovdqa	(%rdi), %xmm4
11153        vmovdqa	(%r9), %xmm5
11154        vmovdqa	(%r8), %xmm6
11155        vpsrlq	$63, %xmm5, %xmm8
11156        vpsllq	$0x01, %xmm5, %xmm7
11157        vpslldq	$8, %xmm8, %xmm8
11158        vpor	%xmm8, %xmm7, %xmm7
11159        vpshufd	$0xff, %xmm5, %xmm5
11160        vpsrad	$31, %xmm5, %xmm5
11161        vpand	L_avx1_aes_gcm_mod2_128(%rip), %xmm5, %xmm5
11162        vpxor	%xmm7, %xmm5, %xmm5
11163        movl	%r10d, %edx
11164        movl	%r11d, %ecx
11165        shlq	$3, %rdx
11166        shlq	$3, %rcx
11167        vmovq	%rdx, %xmm0
11168        vmovq	%rcx, %xmm1
11169        vpunpcklqdq	%xmm1, %xmm0, %xmm0
11170        vpxor	%xmm0, %xmm4, %xmm4
11171        # ghash_gfmul_red_avx
11172        vpshufd	$0x4e, %xmm5, %xmm8
11173        vpshufd	$0x4e, %xmm4, %xmm9
11174        vpclmulqdq	$0x11, %xmm5, %xmm4, %xmm10
11175        vpclmulqdq	$0x00, %xmm5, %xmm4, %xmm7
11176        vpxor	%xmm5, %xmm8, %xmm8
11177        vpxor	%xmm4, %xmm9, %xmm9
11178        vpclmulqdq	$0x00, %xmm9, %xmm8, %xmm8
11179        vpxor	%xmm7, %xmm8, %xmm8
11180        vpxor	%xmm10, %xmm8, %xmm8
11181        vpslldq	$8, %xmm8, %xmm9
11182        vpsrldq	$8, %xmm8, %xmm8
11183        vpxor	%xmm9, %xmm7, %xmm7
11184        vpxor	%xmm8, %xmm10, %xmm4
11185        vpslld	$31, %xmm7, %xmm11
11186        vpslld	$30, %xmm7, %xmm12
11187        vpslld	$25, %xmm7, %xmm13
11188        vpxor	%xmm12, %xmm11, %xmm11
11189        vpxor	%xmm13, %xmm11, %xmm11
11190        vpsrldq	$4, %xmm11, %xmm12
11191        vpslldq	$12, %xmm11, %xmm11
11192        vpxor	%xmm11, %xmm7, %xmm7
11193        vpsrld	$0x01, %xmm7, %xmm13
11194        vpsrld	$2, %xmm7, %xmm9
11195        vpsrld	$7, %xmm7, %xmm8
11196        vpxor	%xmm9, %xmm13, %xmm13
11197        vpxor	%xmm8, %xmm13, %xmm13
11198        vpxor	%xmm12, %xmm13, %xmm13
11199        vpxor	%xmm7, %xmm13, %xmm13
11200        vpxor	%xmm13, %xmm4, %xmm4
11201        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm4, %xmm4
11202        vpxor	%xmm6, %xmm4, %xmm0
11203        cmpl	$16, %eax
11204        je	L_AES_GCM_encrypt_final_avx1_store_tag_16
11205        xorq	%rcx, %rcx
11206        vmovdqu	%xmm0, (%rsp)
11207L_AES_GCM_encrypt_final_avx1_store_tag_loop:
11208        movzbl	(%rsp,%rcx,1), %r13d
11209        movb	%r13b, (%rsi,%rcx,1)
11210        incl	%ecx
11211        cmpl	%eax, %ecx
11212        jne	L_AES_GCM_encrypt_final_avx1_store_tag_loop
11213        jmp	L_AES_GCM_encrypt_final_avx1_store_tag_done
11214L_AES_GCM_encrypt_final_avx1_store_tag_16:
11215        vmovdqu	%xmm0, (%rsi)
11216L_AES_GCM_encrypt_final_avx1_store_tag_done:
11217        vzeroupper
11218        addq	$16, %rsp
11219        popq	%r13
11220        repz retq
11221#ifndef __APPLE__
11222.size	AES_GCM_encrypt_final_avx1,.-AES_GCM_encrypt_final_avx1
11223#endif /* __APPLE__ */
11224#ifndef __APPLE__
11225.text
11226.globl	AES_GCM_decrypt_update_avx1
11227.type	AES_GCM_decrypt_update_avx1,@function
11228.align	16
11229AES_GCM_decrypt_update_avx1:
11230#else
11231.section	__TEXT,__text
11232.globl	_AES_GCM_decrypt_update_avx1
11233.p2align	4
11234_AES_GCM_decrypt_update_avx1:
11235#endif /* __APPLE__ */
11236        pushq	%r13
11237        pushq	%r12
11238        pushq	%r14
11239        movq	%rdx, %r10
11240        movq	%rcx, %r11
11241        movq	32(%rsp), %rax
11242        movq	40(%rsp), %r12
11243        subq	$0xa8, %rsp
11244        vmovdqa	(%r9), %xmm6
11245        vmovdqa	(%rax), %xmm5
11246        vpsrlq	$63, %xmm5, %xmm9
11247        vpsllq	$0x01, %xmm5, %xmm8
11248        vpslldq	$8, %xmm9, %xmm9
11249        vpor	%xmm9, %xmm8, %xmm8
11250        vpshufd	$0xff, %xmm5, %xmm5
11251        vpsrad	$31, %xmm5, %xmm5
11252        vpand	L_avx1_aes_gcm_mod2_128(%rip), %xmm5, %xmm5
11253        vpxor	%xmm8, %xmm5, %xmm5
11254        xorl	%r14d, %r14d
11255        cmpl	$0x80, %r8d
11256        movl	%r8d, %r13d
11257        jl	L_AES_GCM_decrypt_update_avx1_done_128
11258        andl	$0xffffff80, %r13d
11259        vmovdqa	%xmm6, %xmm2
11260        # H ^ 1
11261        vmovdqu	%xmm5, (%rsp)
11262        # H ^ 2
11263        vpclmulqdq	$0x00, %xmm5, %xmm5, %xmm8
11264        vpclmulqdq	$0x11, %xmm5, %xmm5, %xmm0
11265        vpslld	$31, %xmm8, %xmm12
11266        vpslld	$30, %xmm8, %xmm13
11267        vpslld	$25, %xmm8, %xmm14
11268        vpxor	%xmm13, %xmm12, %xmm12
11269        vpxor	%xmm14, %xmm12, %xmm12
11270        vpsrldq	$4, %xmm12, %xmm13
11271        vpslldq	$12, %xmm12, %xmm12
11272        vpxor	%xmm12, %xmm8, %xmm8
11273        vpsrld	$0x01, %xmm8, %xmm14
11274        vpsrld	$2, %xmm8, %xmm10
11275        vpsrld	$7, %xmm8, %xmm9
11276        vpxor	%xmm10, %xmm14, %xmm14
11277        vpxor	%xmm9, %xmm14, %xmm14
11278        vpxor	%xmm13, %xmm14, %xmm14
11279        vpxor	%xmm8, %xmm14, %xmm14
11280        vpxor	%xmm14, %xmm0, %xmm0
11281        vmovdqu	%xmm0, 16(%rsp)
11282        # H ^ 3
11283        # ghash_gfmul_red_avx
11284        vpshufd	$0x4e, %xmm5, %xmm9
11285        vpshufd	$0x4e, %xmm0, %xmm10
11286        vpclmulqdq	$0x11, %xmm5, %xmm0, %xmm11
11287        vpclmulqdq	$0x00, %xmm5, %xmm0, %xmm8
11288        vpxor	%xmm5, %xmm9, %xmm9
11289        vpxor	%xmm0, %xmm10, %xmm10
11290        vpclmulqdq	$0x00, %xmm10, %xmm9, %xmm9
11291        vpxor	%xmm8, %xmm9, %xmm9
11292        vpxor	%xmm11, %xmm9, %xmm9
11293        vpslldq	$8, %xmm9, %xmm10
11294        vpsrldq	$8, %xmm9, %xmm9
11295        vpxor	%xmm10, %xmm8, %xmm8
11296        vpxor	%xmm9, %xmm11, %xmm1
11297        vpslld	$31, %xmm8, %xmm12
11298        vpslld	$30, %xmm8, %xmm13
11299        vpslld	$25, %xmm8, %xmm14
11300        vpxor	%xmm13, %xmm12, %xmm12
11301        vpxor	%xmm14, %xmm12, %xmm12
11302        vpsrldq	$4, %xmm12, %xmm13
11303        vpslldq	$12, %xmm12, %xmm12
11304        vpxor	%xmm12, %xmm8, %xmm8
11305        vpsrld	$0x01, %xmm8, %xmm14
11306        vpsrld	$2, %xmm8, %xmm10
11307        vpsrld	$7, %xmm8, %xmm9
11308        vpxor	%xmm10, %xmm14, %xmm14
11309        vpxor	%xmm9, %xmm14, %xmm14
11310        vpxor	%xmm13, %xmm14, %xmm14
11311        vpxor	%xmm8, %xmm14, %xmm14
11312        vpxor	%xmm14, %xmm1, %xmm1
11313        vmovdqu	%xmm1, 32(%rsp)
11314        # H ^ 4
11315        vpclmulqdq	$0x00, %xmm0, %xmm0, %xmm8
11316        vpclmulqdq	$0x11, %xmm0, %xmm0, %xmm3
11317        vpslld	$31, %xmm8, %xmm12
11318        vpslld	$30, %xmm8, %xmm13
11319        vpslld	$25, %xmm8, %xmm14
11320        vpxor	%xmm13, %xmm12, %xmm12
11321        vpxor	%xmm14, %xmm12, %xmm12
11322        vpsrldq	$4, %xmm12, %xmm13
11323        vpslldq	$12, %xmm12, %xmm12
11324        vpxor	%xmm12, %xmm8, %xmm8
11325        vpsrld	$0x01, %xmm8, %xmm14
11326        vpsrld	$2, %xmm8, %xmm10
11327        vpsrld	$7, %xmm8, %xmm9
11328        vpxor	%xmm10, %xmm14, %xmm14
11329        vpxor	%xmm9, %xmm14, %xmm14
11330        vpxor	%xmm13, %xmm14, %xmm14
11331        vpxor	%xmm8, %xmm14, %xmm14
11332        vpxor	%xmm14, %xmm3, %xmm3
11333        vmovdqu	%xmm3, 48(%rsp)
11334        # H ^ 5
11335        # ghash_gfmul_red_avx
11336        vpshufd	$0x4e, %xmm0, %xmm9
11337        vpshufd	$0x4e, %xmm1, %xmm10
11338        vpclmulqdq	$0x11, %xmm0, %xmm1, %xmm11
11339        vpclmulqdq	$0x00, %xmm0, %xmm1, %xmm8
11340        vpxor	%xmm0, %xmm9, %xmm9
11341        vpxor	%xmm1, %xmm10, %xmm10
11342        vpclmulqdq	$0x00, %xmm10, %xmm9, %xmm9
11343        vpxor	%xmm8, %xmm9, %xmm9
11344        vpxor	%xmm11, %xmm9, %xmm9
11345        vpslldq	$8, %xmm9, %xmm10
11346        vpsrldq	$8, %xmm9, %xmm9
11347        vpxor	%xmm10, %xmm8, %xmm8
11348        vpxor	%xmm9, %xmm11, %xmm7
11349        vpslld	$31, %xmm8, %xmm12
11350        vpslld	$30, %xmm8, %xmm13
11351        vpslld	$25, %xmm8, %xmm14
11352        vpxor	%xmm13, %xmm12, %xmm12
11353        vpxor	%xmm14, %xmm12, %xmm12
11354        vpsrldq	$4, %xmm12, %xmm13
11355        vpslldq	$12, %xmm12, %xmm12
11356        vpxor	%xmm12, %xmm8, %xmm8
11357        vpsrld	$0x01, %xmm8, %xmm14
11358        vpsrld	$2, %xmm8, %xmm10
11359        vpsrld	$7, %xmm8, %xmm9
11360        vpxor	%xmm10, %xmm14, %xmm14
11361        vpxor	%xmm9, %xmm14, %xmm14
11362        vpxor	%xmm13, %xmm14, %xmm14
11363        vpxor	%xmm8, %xmm14, %xmm14
11364        vpxor	%xmm14, %xmm7, %xmm7
11365        vmovdqu	%xmm7, 64(%rsp)
11366        # H ^ 6
11367        vpclmulqdq	$0x00, %xmm1, %xmm1, %xmm8
11368        vpclmulqdq	$0x11, %xmm1, %xmm1, %xmm7
11369        vpslld	$31, %xmm8, %xmm12
11370        vpslld	$30, %xmm8, %xmm13
11371        vpslld	$25, %xmm8, %xmm14
11372        vpxor	%xmm13, %xmm12, %xmm12
11373        vpxor	%xmm14, %xmm12, %xmm12
11374        vpsrldq	$4, %xmm12, %xmm13
11375        vpslldq	$12, %xmm12, %xmm12
11376        vpxor	%xmm12, %xmm8, %xmm8
11377        vpsrld	$0x01, %xmm8, %xmm14
11378        vpsrld	$2, %xmm8, %xmm10
11379        vpsrld	$7, %xmm8, %xmm9
11380        vpxor	%xmm10, %xmm14, %xmm14
11381        vpxor	%xmm9, %xmm14, %xmm14
11382        vpxor	%xmm13, %xmm14, %xmm14
11383        vpxor	%xmm8, %xmm14, %xmm14
11384        vpxor	%xmm14, %xmm7, %xmm7
11385        vmovdqu	%xmm7, 80(%rsp)
11386        # H ^ 7
11387        # ghash_gfmul_red_avx
11388        vpshufd	$0x4e, %xmm1, %xmm9
11389        vpshufd	$0x4e, %xmm3, %xmm10
11390        vpclmulqdq	$0x11, %xmm1, %xmm3, %xmm11
11391        vpclmulqdq	$0x00, %xmm1, %xmm3, %xmm8
11392        vpxor	%xmm1, %xmm9, %xmm9
11393        vpxor	%xmm3, %xmm10, %xmm10
11394        vpclmulqdq	$0x00, %xmm10, %xmm9, %xmm9
11395        vpxor	%xmm8, %xmm9, %xmm9
11396        vpxor	%xmm11, %xmm9, %xmm9
11397        vpslldq	$8, %xmm9, %xmm10
11398        vpsrldq	$8, %xmm9, %xmm9
11399        vpxor	%xmm10, %xmm8, %xmm8
11400        vpxor	%xmm9, %xmm11, %xmm7
11401        vpslld	$31, %xmm8, %xmm12
11402        vpslld	$30, %xmm8, %xmm13
11403        vpslld	$25, %xmm8, %xmm14
11404        vpxor	%xmm13, %xmm12, %xmm12
11405        vpxor	%xmm14, %xmm12, %xmm12
11406        vpsrldq	$4, %xmm12, %xmm13
11407        vpslldq	$12, %xmm12, %xmm12
11408        vpxor	%xmm12, %xmm8, %xmm8
11409        vpsrld	$0x01, %xmm8, %xmm14
11410        vpsrld	$2, %xmm8, %xmm10
11411        vpsrld	$7, %xmm8, %xmm9
11412        vpxor	%xmm10, %xmm14, %xmm14
11413        vpxor	%xmm9, %xmm14, %xmm14
11414        vpxor	%xmm13, %xmm14, %xmm14
11415        vpxor	%xmm8, %xmm14, %xmm14
11416        vpxor	%xmm14, %xmm7, %xmm7
11417        vmovdqu	%xmm7, 96(%rsp)
11418        # H ^ 8
11419        vpclmulqdq	$0x00, %xmm3, %xmm3, %xmm8
11420        vpclmulqdq	$0x11, %xmm3, %xmm3, %xmm7
11421        vpslld	$31, %xmm8, %xmm12
11422        vpslld	$30, %xmm8, %xmm13
11423        vpslld	$25, %xmm8, %xmm14
11424        vpxor	%xmm13, %xmm12, %xmm12
11425        vpxor	%xmm14, %xmm12, %xmm12
11426        vpsrldq	$4, %xmm12, %xmm13
11427        vpslldq	$12, %xmm12, %xmm12
11428        vpxor	%xmm12, %xmm8, %xmm8
11429        vpsrld	$0x01, %xmm8, %xmm14
11430        vpsrld	$2, %xmm8, %xmm10
11431        vpsrld	$7, %xmm8, %xmm9
11432        vpxor	%xmm10, %xmm14, %xmm14
11433        vpxor	%xmm9, %xmm14, %xmm14
11434        vpxor	%xmm13, %xmm14, %xmm14
11435        vpxor	%xmm8, %xmm14, %xmm14
11436        vpxor	%xmm14, %xmm7, %xmm7
11437        vmovdqu	%xmm7, 112(%rsp)
11438L_AES_GCM_decrypt_update_avx1_ghash_128:
11439        leaq	(%r11,%r14,1), %rcx
11440        leaq	(%r10,%r14,1), %rdx
11441        vmovdqu	(%r12), %xmm0
11442        vmovdqa	L_avx1_aes_gcm_bswap_epi64(%rip), %xmm1
11443        vpshufb	%xmm1, %xmm0, %xmm8
11444        vpaddd	L_avx1_aes_gcm_one(%rip), %xmm0, %xmm9
11445        vpshufb	%xmm1, %xmm9, %xmm9
11446        vpaddd	L_avx1_aes_gcm_two(%rip), %xmm0, %xmm10
11447        vpshufb	%xmm1, %xmm10, %xmm10
11448        vpaddd	L_avx1_aes_gcm_three(%rip), %xmm0, %xmm11
11449        vpshufb	%xmm1, %xmm11, %xmm11
11450        vpaddd	L_avx1_aes_gcm_four(%rip), %xmm0, %xmm12
11451        vpshufb	%xmm1, %xmm12, %xmm12
11452        vpaddd	L_avx1_aes_gcm_five(%rip), %xmm0, %xmm13
11453        vpshufb	%xmm1, %xmm13, %xmm13
11454        vpaddd	L_avx1_aes_gcm_six(%rip), %xmm0, %xmm14
11455        vpshufb	%xmm1, %xmm14, %xmm14
11456        vpaddd	L_avx1_aes_gcm_seven(%rip), %xmm0, %xmm15
11457        vpshufb	%xmm1, %xmm15, %xmm15
11458        vpaddd	L_avx1_aes_gcm_eight(%rip), %xmm0, %xmm0
11459        vmovdqa	(%rdi), %xmm7
11460        vmovdqu	%xmm0, (%r12)
11461        vpxor	%xmm7, %xmm8, %xmm8
11462        vpxor	%xmm7, %xmm9, %xmm9
11463        vpxor	%xmm7, %xmm10, %xmm10
11464        vpxor	%xmm7, %xmm11, %xmm11
11465        vpxor	%xmm7, %xmm12, %xmm12
11466        vpxor	%xmm7, %xmm13, %xmm13
11467        vpxor	%xmm7, %xmm14, %xmm14
11468        vpxor	%xmm7, %xmm15, %xmm15
11469        vmovdqu	112(%rsp), %xmm7
11470        vmovdqu	(%rcx), %xmm0
11471        vaesenc	16(%rdi), %xmm8, %xmm8
11472        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0
11473        vpxor	%xmm2, %xmm0, %xmm0
11474        vpshufd	$0x4e, %xmm7, %xmm1
11475        vpshufd	$0x4e, %xmm0, %xmm5
11476        vpxor	%xmm7, %xmm1, %xmm1
11477        vpxor	%xmm0, %xmm5, %xmm5
11478        vpclmulqdq	$0x11, %xmm7, %xmm0, %xmm3
11479        vaesenc	16(%rdi), %xmm9, %xmm9
11480        vaesenc	16(%rdi), %xmm10, %xmm10
11481        vpclmulqdq	$0x00, %xmm7, %xmm0, %xmm2
11482        vaesenc	16(%rdi), %xmm11, %xmm11
11483        vaesenc	16(%rdi), %xmm12, %xmm12
11484        vpclmulqdq	$0x00, %xmm5, %xmm1, %xmm1
11485        vaesenc	16(%rdi), %xmm13, %xmm13
11486        vaesenc	16(%rdi), %xmm14, %xmm14
11487        vaesenc	16(%rdi), %xmm15, %xmm15
11488        vpxor	%xmm2, %xmm1, %xmm1
11489        vpxor	%xmm3, %xmm1, %xmm1
11490        vmovdqu	96(%rsp), %xmm7
11491        vmovdqu	16(%rcx), %xmm0
11492        vpshufd	$0x4e, %xmm7, %xmm4
11493        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0
11494        vaesenc	32(%rdi), %xmm8, %xmm8
11495        vpxor	%xmm7, %xmm4, %xmm4
11496        vpshufd	$0x4e, %xmm0, %xmm5
11497        vpxor	%xmm0, %xmm5, %xmm5
11498        vpclmulqdq	$0x11, %xmm7, %xmm0, %xmm6
11499        vaesenc	32(%rdi), %xmm9, %xmm9
11500        vaesenc	32(%rdi), %xmm10, %xmm10
11501        vpclmulqdq	$0x00, %xmm7, %xmm0, %xmm7
11502        vaesenc	32(%rdi), %xmm11, %xmm11
11503        vaesenc	32(%rdi), %xmm12, %xmm12
11504        vpclmulqdq	$0x00, %xmm5, %xmm4, %xmm4
11505        vaesenc	32(%rdi), %xmm13, %xmm13
11506        vaesenc	32(%rdi), %xmm14, %xmm14
11507        vaesenc	32(%rdi), %xmm15, %xmm15
11508        vpxor	%xmm7, %xmm1, %xmm1
11509        vpxor	%xmm7, %xmm2, %xmm2
11510        vpxor	%xmm6, %xmm1, %xmm1
11511        vpxor	%xmm6, %xmm3, %xmm3
11512        vpxor	%xmm4, %xmm1, %xmm1
11513        vmovdqu	80(%rsp), %xmm7
11514        vmovdqu	32(%rcx), %xmm0
11515        vpshufd	$0x4e, %xmm7, %xmm4
11516        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0
11517        vaesenc	48(%rdi), %xmm8, %xmm8
11518        vpxor	%xmm7, %xmm4, %xmm4
11519        vpshufd	$0x4e, %xmm0, %xmm5
11520        vpxor	%xmm0, %xmm5, %xmm5
11521        vpclmulqdq	$0x11, %xmm7, %xmm0, %xmm6
11522        vaesenc	48(%rdi), %xmm9, %xmm9
11523        vaesenc	48(%rdi), %xmm10, %xmm10
11524        vpclmulqdq	$0x00, %xmm7, %xmm0, %xmm7
11525        vaesenc	48(%rdi), %xmm11, %xmm11
11526        vaesenc	48(%rdi), %xmm12, %xmm12
11527        vpclmulqdq	$0x00, %xmm5, %xmm4, %xmm4
11528        vaesenc	48(%rdi), %xmm13, %xmm13
11529        vaesenc	48(%rdi), %xmm14, %xmm14
11530        vaesenc	48(%rdi), %xmm15, %xmm15
11531        vpxor	%xmm7, %xmm1, %xmm1
11532        vpxor	%xmm7, %xmm2, %xmm2
11533        vpxor	%xmm6, %xmm1, %xmm1
11534        vpxor	%xmm6, %xmm3, %xmm3
11535        vpxor	%xmm4, %xmm1, %xmm1
11536        vmovdqu	64(%rsp), %xmm7
11537        vmovdqu	48(%rcx), %xmm0
11538        vpshufd	$0x4e, %xmm7, %xmm4
11539        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0
11540        vaesenc	64(%rdi), %xmm8, %xmm8
11541        vpxor	%xmm7, %xmm4, %xmm4
11542        vpshufd	$0x4e, %xmm0, %xmm5
11543        vpxor	%xmm0, %xmm5, %xmm5
11544        vpclmulqdq	$0x11, %xmm7, %xmm0, %xmm6
11545        vaesenc	64(%rdi), %xmm9, %xmm9
11546        vaesenc	64(%rdi), %xmm10, %xmm10
11547        vpclmulqdq	$0x00, %xmm7, %xmm0, %xmm7
11548        vaesenc	64(%rdi), %xmm11, %xmm11
11549        vaesenc	64(%rdi), %xmm12, %xmm12
11550        vpclmulqdq	$0x00, %xmm5, %xmm4, %xmm4
11551        vaesenc	64(%rdi), %xmm13, %xmm13
11552        vaesenc	64(%rdi), %xmm14, %xmm14
11553        vaesenc	64(%rdi), %xmm15, %xmm15
11554        vpxor	%xmm7, %xmm1, %xmm1
11555        vpxor	%xmm7, %xmm2, %xmm2
11556        vpxor	%xmm6, %xmm1, %xmm1
11557        vpxor	%xmm6, %xmm3, %xmm3
11558        vpxor	%xmm4, %xmm1, %xmm1
11559        vmovdqu	48(%rsp), %xmm7
11560        vmovdqu	64(%rcx), %xmm0
11561        vpshufd	$0x4e, %xmm7, %xmm4
11562        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0
11563        vaesenc	80(%rdi), %xmm8, %xmm8
11564        vpxor	%xmm7, %xmm4, %xmm4
11565        vpshufd	$0x4e, %xmm0, %xmm5
11566        vpxor	%xmm0, %xmm5, %xmm5
11567        vpclmulqdq	$0x11, %xmm7, %xmm0, %xmm6
11568        vaesenc	80(%rdi), %xmm9, %xmm9
11569        vaesenc	80(%rdi), %xmm10, %xmm10
11570        vpclmulqdq	$0x00, %xmm7, %xmm0, %xmm7
11571        vaesenc	80(%rdi), %xmm11, %xmm11
11572        vaesenc	80(%rdi), %xmm12, %xmm12
11573        vpclmulqdq	$0x00, %xmm5, %xmm4, %xmm4
11574        vaesenc	80(%rdi), %xmm13, %xmm13
11575        vaesenc	80(%rdi), %xmm14, %xmm14
11576        vaesenc	80(%rdi), %xmm15, %xmm15
11577        vpxor	%xmm7, %xmm1, %xmm1
11578        vpxor	%xmm7, %xmm2, %xmm2
11579        vpxor	%xmm6, %xmm1, %xmm1
11580        vpxor	%xmm6, %xmm3, %xmm3
11581        vpxor	%xmm4, %xmm1, %xmm1
11582        vmovdqu	32(%rsp), %xmm7
11583        vmovdqu	80(%rcx), %xmm0
11584        vpshufd	$0x4e, %xmm7, %xmm4
11585        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0
11586        vaesenc	96(%rdi), %xmm8, %xmm8
11587        vpxor	%xmm7, %xmm4, %xmm4
11588        vpshufd	$0x4e, %xmm0, %xmm5
11589        vpxor	%xmm0, %xmm5, %xmm5
11590        vpclmulqdq	$0x11, %xmm7, %xmm0, %xmm6
11591        vaesenc	96(%rdi), %xmm9, %xmm9
11592        vaesenc	96(%rdi), %xmm10, %xmm10
11593        vpclmulqdq	$0x00, %xmm7, %xmm0, %xmm7
11594        vaesenc	96(%rdi), %xmm11, %xmm11
11595        vaesenc	96(%rdi), %xmm12, %xmm12
11596        vpclmulqdq	$0x00, %xmm5, %xmm4, %xmm4
11597        vaesenc	96(%rdi), %xmm13, %xmm13
11598        vaesenc	96(%rdi), %xmm14, %xmm14
11599        vaesenc	96(%rdi), %xmm15, %xmm15
11600        vpxor	%xmm7, %xmm1, %xmm1
11601        vpxor	%xmm7, %xmm2, %xmm2
11602        vpxor	%xmm6, %xmm1, %xmm1
11603        vpxor	%xmm6, %xmm3, %xmm3
11604        vpxor	%xmm4, %xmm1, %xmm1
11605        vmovdqu	16(%rsp), %xmm7
11606        vmovdqu	96(%rcx), %xmm0
11607        vpshufd	$0x4e, %xmm7, %xmm4
11608        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0
11609        vaesenc	112(%rdi), %xmm8, %xmm8
11610        vpxor	%xmm7, %xmm4, %xmm4
11611        vpshufd	$0x4e, %xmm0, %xmm5
11612        vpxor	%xmm0, %xmm5, %xmm5
11613        vpclmulqdq	$0x11, %xmm7, %xmm0, %xmm6
11614        vaesenc	112(%rdi), %xmm9, %xmm9
11615        vaesenc	112(%rdi), %xmm10, %xmm10
11616        vpclmulqdq	$0x00, %xmm7, %xmm0, %xmm7
11617        vaesenc	112(%rdi), %xmm11, %xmm11
11618        vaesenc	112(%rdi), %xmm12, %xmm12
11619        vpclmulqdq	$0x00, %xmm5, %xmm4, %xmm4
11620        vaesenc	112(%rdi), %xmm13, %xmm13
11621        vaesenc	112(%rdi), %xmm14, %xmm14
11622        vaesenc	112(%rdi), %xmm15, %xmm15
11623        vpxor	%xmm7, %xmm1, %xmm1
11624        vpxor	%xmm7, %xmm2, %xmm2
11625        vpxor	%xmm6, %xmm1, %xmm1
11626        vpxor	%xmm6, %xmm3, %xmm3
11627        vpxor	%xmm4, %xmm1, %xmm1
11628        vmovdqu	(%rsp), %xmm7
11629        vmovdqu	112(%rcx), %xmm0
11630        vpshufd	$0x4e, %xmm7, %xmm4
11631        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0
11632        vaesenc	128(%rdi), %xmm8, %xmm8
11633        vpxor	%xmm7, %xmm4, %xmm4
11634        vpshufd	$0x4e, %xmm0, %xmm5
11635        vpxor	%xmm0, %xmm5, %xmm5
11636        vpclmulqdq	$0x11, %xmm7, %xmm0, %xmm6
11637        vaesenc	128(%rdi), %xmm9, %xmm9
11638        vaesenc	128(%rdi), %xmm10, %xmm10
11639        vpclmulqdq	$0x00, %xmm7, %xmm0, %xmm7
11640        vaesenc	128(%rdi), %xmm11, %xmm11
11641        vaesenc	128(%rdi), %xmm12, %xmm12
11642        vpclmulqdq	$0x00, %xmm5, %xmm4, %xmm4
11643        vaesenc	128(%rdi), %xmm13, %xmm13
11644        vaesenc	128(%rdi), %xmm14, %xmm14
11645        vaesenc	128(%rdi), %xmm15, %xmm15
11646        vpxor	%xmm7, %xmm1, %xmm1
11647        vpxor	%xmm7, %xmm2, %xmm2
11648        vpxor	%xmm6, %xmm1, %xmm1
11649        vpxor	%xmm6, %xmm3, %xmm3
11650        vpxor	%xmm4, %xmm1, %xmm1
11651        vpslldq	$8, %xmm1, %xmm5
11652        vpsrldq	$8, %xmm1, %xmm1
11653        vaesenc	144(%rdi), %xmm8, %xmm8
11654        vpxor	%xmm5, %xmm2, %xmm2
11655        vpxor	%xmm1, %xmm3, %xmm3
11656        vaesenc	144(%rdi), %xmm9, %xmm9
11657        vpslld	$31, %xmm2, %xmm7
11658        vpslld	$30, %xmm2, %xmm4
11659        vpslld	$25, %xmm2, %xmm5
11660        vaesenc	144(%rdi), %xmm10, %xmm10
11661        vpxor	%xmm4, %xmm7, %xmm7
11662        vpxor	%xmm5, %xmm7, %xmm7
11663        vaesenc	144(%rdi), %xmm11, %xmm11
11664        vpsrldq	$4, %xmm7, %xmm4
11665        vpslldq	$12, %xmm7, %xmm7
11666        vaesenc	144(%rdi), %xmm12, %xmm12
11667        vpxor	%xmm7, %xmm2, %xmm2
11668        vpsrld	$0x01, %xmm2, %xmm5
11669        vaesenc	144(%rdi), %xmm13, %xmm13
11670        vpsrld	$2, %xmm2, %xmm1
11671        vpsrld	$7, %xmm2, %xmm0
11672        vaesenc	144(%rdi), %xmm14, %xmm14
11673        vpxor	%xmm1, %xmm5, %xmm5
11674        vpxor	%xmm0, %xmm5, %xmm5
11675        vaesenc	144(%rdi), %xmm15, %xmm15
11676        vpxor	%xmm4, %xmm5, %xmm5
11677        vpxor	%xmm5, %xmm2, %xmm2
11678        vpxor	%xmm3, %xmm2, %xmm2
11679        cmpl	$11, %esi
11680        vmovdqa	160(%rdi), %xmm7
11681        jl	L_AES_GCM_decrypt_update_avx1_aesenc_128_ghash_avx_done
11682        vaesenc	%xmm7, %xmm8, %xmm8
11683        vaesenc	%xmm7, %xmm9, %xmm9
11684        vaesenc	%xmm7, %xmm10, %xmm10
11685        vaesenc	%xmm7, %xmm11, %xmm11
11686        vaesenc	%xmm7, %xmm12, %xmm12
11687        vaesenc	%xmm7, %xmm13, %xmm13
11688        vaesenc	%xmm7, %xmm14, %xmm14
11689        vaesenc	%xmm7, %xmm15, %xmm15
11690        vmovdqa	176(%rdi), %xmm7
11691        vaesenc	%xmm7, %xmm8, %xmm8
11692        vaesenc	%xmm7, %xmm9, %xmm9
11693        vaesenc	%xmm7, %xmm10, %xmm10
11694        vaesenc	%xmm7, %xmm11, %xmm11
11695        vaesenc	%xmm7, %xmm12, %xmm12
11696        vaesenc	%xmm7, %xmm13, %xmm13
11697        vaesenc	%xmm7, %xmm14, %xmm14
11698        vaesenc	%xmm7, %xmm15, %xmm15
11699        cmpl	$13, %esi
11700        vmovdqa	192(%rdi), %xmm7
11701        jl	L_AES_GCM_decrypt_update_avx1_aesenc_128_ghash_avx_done
11702        vaesenc	%xmm7, %xmm8, %xmm8
11703        vaesenc	%xmm7, %xmm9, %xmm9
11704        vaesenc	%xmm7, %xmm10, %xmm10
11705        vaesenc	%xmm7, %xmm11, %xmm11
11706        vaesenc	%xmm7, %xmm12, %xmm12
11707        vaesenc	%xmm7, %xmm13, %xmm13
11708        vaesenc	%xmm7, %xmm14, %xmm14
11709        vaesenc	%xmm7, %xmm15, %xmm15
11710        vmovdqa	208(%rdi), %xmm7
11711        vaesenc	%xmm7, %xmm8, %xmm8
11712        vaesenc	%xmm7, %xmm9, %xmm9
11713        vaesenc	%xmm7, %xmm10, %xmm10
11714        vaesenc	%xmm7, %xmm11, %xmm11
11715        vaesenc	%xmm7, %xmm12, %xmm12
11716        vaesenc	%xmm7, %xmm13, %xmm13
11717        vaesenc	%xmm7, %xmm14, %xmm14
11718        vaesenc	%xmm7, %xmm15, %xmm15
11719        vmovdqa	224(%rdi), %xmm7
11720L_AES_GCM_decrypt_update_avx1_aesenc_128_ghash_avx_done:
11721        vaesenclast	%xmm7, %xmm8, %xmm8
11722        vaesenclast	%xmm7, %xmm9, %xmm9
11723        vmovdqu	(%rcx), %xmm0
11724        vmovdqu	16(%rcx), %xmm1
11725        vpxor	%xmm0, %xmm8, %xmm8
11726        vpxor	%xmm1, %xmm9, %xmm9
11727        vmovdqu	%xmm8, (%rdx)
11728        vmovdqu	%xmm9, 16(%rdx)
11729        vaesenclast	%xmm7, %xmm10, %xmm10
11730        vaesenclast	%xmm7, %xmm11, %xmm11
11731        vmovdqu	32(%rcx), %xmm0
11732        vmovdqu	48(%rcx), %xmm1
11733        vpxor	%xmm0, %xmm10, %xmm10
11734        vpxor	%xmm1, %xmm11, %xmm11
11735        vmovdqu	%xmm10, 32(%rdx)
11736        vmovdqu	%xmm11, 48(%rdx)
11737        vaesenclast	%xmm7, %xmm12, %xmm12
11738        vaesenclast	%xmm7, %xmm13, %xmm13
11739        vmovdqu	64(%rcx), %xmm0
11740        vmovdqu	80(%rcx), %xmm1
11741        vpxor	%xmm0, %xmm12, %xmm12
11742        vpxor	%xmm1, %xmm13, %xmm13
11743        vmovdqu	%xmm12, 64(%rdx)
11744        vmovdqu	%xmm13, 80(%rdx)
11745        vaesenclast	%xmm7, %xmm14, %xmm14
11746        vaesenclast	%xmm7, %xmm15, %xmm15
11747        vmovdqu	96(%rcx), %xmm0
11748        vmovdqu	112(%rcx), %xmm1
11749        vpxor	%xmm0, %xmm14, %xmm14
11750        vpxor	%xmm1, %xmm15, %xmm15
11751        vmovdqu	%xmm14, 96(%rdx)
11752        vmovdqu	%xmm15, 112(%rdx)
11753        addl	$0x80, %r14d
11754        cmpl	%r13d, %r14d
11755        jl	L_AES_GCM_decrypt_update_avx1_ghash_128
11756        vmovdqa	%xmm2, %xmm6
11757        vmovdqu	(%rsp), %xmm5
11758L_AES_GCM_decrypt_update_avx1_done_128:
11759        movl	%r8d, %edx
11760        cmpl	%edx, %r14d
11761        jge	L_AES_GCM_decrypt_update_avx1_done_dec
11762        movl	%r8d, %r13d
11763        andl	$0xfffffff0, %r13d
11764        cmpl	%r13d, %r14d
11765        jge	L_AES_GCM_decrypt_update_avx1_last_block_done
11766L_AES_GCM_decrypt_update_avx1_last_block_start:
11767        vmovdqu	(%r11,%r14,1), %xmm13
11768        vmovdqa	%xmm5, %xmm0
11769        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm13, %xmm1
11770        vpxor	%xmm6, %xmm1, %xmm1
11771        vmovdqu	(%r12), %xmm9
11772        vpshufb	L_avx1_aes_gcm_bswap_epi64(%rip), %xmm9, %xmm8
11773        vpaddd	L_avx1_aes_gcm_one(%rip), %xmm9, %xmm9
11774        vmovdqu	%xmm9, (%r12)
11775        vpxor	(%rdi), %xmm8, %xmm8
11776        vpclmulqdq	$16, %xmm0, %xmm1, %xmm10
11777        vaesenc	16(%rdi), %xmm8, %xmm8
11778        vaesenc	32(%rdi), %xmm8, %xmm8
11779        vpclmulqdq	$0x01, %xmm0, %xmm1, %xmm11
11780        vaesenc	48(%rdi), %xmm8, %xmm8
11781        vaesenc	64(%rdi), %xmm8, %xmm8
11782        vpclmulqdq	$0x00, %xmm0, %xmm1, %xmm12
11783        vaesenc	80(%rdi), %xmm8, %xmm8
11784        vpclmulqdq	$0x11, %xmm0, %xmm1, %xmm1
11785        vaesenc	96(%rdi), %xmm8, %xmm8
11786        vpxor	%xmm11, %xmm10, %xmm10
11787        vpslldq	$8, %xmm10, %xmm2
11788        vpsrldq	$8, %xmm10, %xmm10
11789        vaesenc	112(%rdi), %xmm8, %xmm8
11790        vpxor	%xmm12, %xmm2, %xmm2
11791        vpxor	%xmm10, %xmm1, %xmm3
11792        vmovdqa	L_avx1_aes_gcm_mod2_128(%rip), %xmm0
11793        vpclmulqdq	$16, %xmm0, %xmm2, %xmm11
11794        vaesenc	128(%rdi), %xmm8, %xmm8
11795        vpshufd	$0x4e, %xmm2, %xmm10
11796        vpxor	%xmm11, %xmm10, %xmm10
11797        vpclmulqdq	$16, %xmm0, %xmm10, %xmm11
11798        vaesenc	144(%rdi), %xmm8, %xmm8
11799        vpshufd	$0x4e, %xmm10, %xmm10
11800        vpxor	%xmm11, %xmm10, %xmm10
11801        vpxor	%xmm3, %xmm10, %xmm6
11802        cmpl	$11, %esi
11803        vmovdqa	160(%rdi), %xmm9
11804        jl	L_AES_GCM_decrypt_update_avx1_aesenc_gfmul_last
11805        vaesenc	%xmm9, %xmm8, %xmm8
11806        vaesenc	176(%rdi), %xmm8, %xmm8
11807        cmpl	$13, %esi
11808        vmovdqa	192(%rdi), %xmm9
11809        jl	L_AES_GCM_decrypt_update_avx1_aesenc_gfmul_last
11810        vaesenc	%xmm9, %xmm8, %xmm8
11811        vaesenc	208(%rdi), %xmm8, %xmm8
11812        vmovdqa	224(%rdi), %xmm9
11813L_AES_GCM_decrypt_update_avx1_aesenc_gfmul_last:
11814        vaesenclast	%xmm9, %xmm8, %xmm8
11815        vmovdqa	%xmm13, %xmm0
11816        vpxor	%xmm0, %xmm8, %xmm8
11817        vmovdqu	%xmm8, (%r10,%r14,1)
11818        addl	$16, %r14d
11819        cmpl	%r13d, %r14d
11820        jl	L_AES_GCM_decrypt_update_avx1_last_block_start
11821L_AES_GCM_decrypt_update_avx1_last_block_done:
11822L_AES_GCM_decrypt_update_avx1_done_dec:
11823        vmovdqa	%xmm6, (%r9)
11824        vzeroupper
11825        addq	$0xa8, %rsp
11826        popq	%r14
11827        popq	%r12
11828        popq	%r13
11829        repz retq
11830#ifndef __APPLE__
11831.size	AES_GCM_decrypt_update_avx1,.-AES_GCM_decrypt_update_avx1
11832#endif /* __APPLE__ */
11833#ifndef __APPLE__
11834.text
11835.globl	AES_GCM_decrypt_final_avx1
11836.type	AES_GCM_decrypt_final_avx1,@function
11837.align	16
11838AES_GCM_decrypt_final_avx1:
11839#else
11840.section	__TEXT,__text
11841.globl	_AES_GCM_decrypt_final_avx1
11842.p2align	4
11843_AES_GCM_decrypt_final_avx1:
11844#endif /* __APPLE__ */
11845        pushq	%r13
11846        pushq	%rbp
11847        pushq	%r12
11848        movl	%edx, %eax
11849        movl	%ecx, %r10d
11850        movl	%r8d, %r11d
11851        movq	32(%rsp), %r8
11852        movq	40(%rsp), %rbp
11853        subq	$16, %rsp
11854        vmovdqa	(%rdi), %xmm6
11855        vmovdqa	(%r9), %xmm5
11856        vmovdqa	(%r8), %xmm15
11857        vpsrlq	$63, %xmm5, %xmm8
11858        vpsllq	$0x01, %xmm5, %xmm7
11859        vpslldq	$8, %xmm8, %xmm8
11860        vpor	%xmm8, %xmm7, %xmm7
11861        vpshufd	$0xff, %xmm5, %xmm5
11862        vpsrad	$31, %xmm5, %xmm5
11863        vpand	L_avx1_aes_gcm_mod2_128(%rip), %xmm5, %xmm5
11864        vpxor	%xmm7, %xmm5, %xmm5
11865        movl	%r10d, %edx
11866        movl	%r11d, %ecx
11867        shlq	$3, %rdx
11868        shlq	$3, %rcx
11869        vmovq	%rdx, %xmm0
11870        vmovq	%rcx, %xmm1
11871        vpunpcklqdq	%xmm1, %xmm0, %xmm0
11872        vpxor	%xmm0, %xmm6, %xmm6
11873        # ghash_gfmul_red_avx
11874        vpshufd	$0x4e, %xmm5, %xmm8
11875        vpshufd	$0x4e, %xmm6, %xmm9
11876        vpclmulqdq	$0x11, %xmm5, %xmm6, %xmm10
11877        vpclmulqdq	$0x00, %xmm5, %xmm6, %xmm7
11878        vpxor	%xmm5, %xmm8, %xmm8
11879        vpxor	%xmm6, %xmm9, %xmm9
11880        vpclmulqdq	$0x00, %xmm9, %xmm8, %xmm8
11881        vpxor	%xmm7, %xmm8, %xmm8
11882        vpxor	%xmm10, %xmm8, %xmm8
11883        vpslldq	$8, %xmm8, %xmm9
11884        vpsrldq	$8, %xmm8, %xmm8
11885        vpxor	%xmm9, %xmm7, %xmm7
11886        vpxor	%xmm8, %xmm10, %xmm6
11887        vpslld	$31, %xmm7, %xmm11
11888        vpslld	$30, %xmm7, %xmm12
11889        vpslld	$25, %xmm7, %xmm13
11890        vpxor	%xmm12, %xmm11, %xmm11
11891        vpxor	%xmm13, %xmm11, %xmm11
11892        vpsrldq	$4, %xmm11, %xmm12
11893        vpslldq	$12, %xmm11, %xmm11
11894        vpxor	%xmm11, %xmm7, %xmm7
11895        vpsrld	$0x01, %xmm7, %xmm13
11896        vpsrld	$2, %xmm7, %xmm9
11897        vpsrld	$7, %xmm7, %xmm8
11898        vpxor	%xmm9, %xmm13, %xmm13
11899        vpxor	%xmm8, %xmm13, %xmm13
11900        vpxor	%xmm12, %xmm13, %xmm13
11901        vpxor	%xmm7, %xmm13, %xmm13
11902        vpxor	%xmm13, %xmm6, %xmm6
11903        vpshufb	L_avx1_aes_gcm_bswap_mask(%rip), %xmm6, %xmm6
11904        vpxor	%xmm15, %xmm6, %xmm0
11905        cmpl	$16, %eax
11906        je	L_AES_GCM_decrypt_final_avx1_cmp_tag_16
11907        subq	$16, %rsp
11908        xorq	%rcx, %rcx
11909        xorq	%r12, %r12
11910        vmovdqu	%xmm0, (%rsp)
11911L_AES_GCM_decrypt_final_avx1_cmp_tag_loop:
11912        movzbl	(%rsp,%rcx,1), %r13d
11913        xorb	(%rsi,%rcx,1), %r13b
11914        orb	%r13b, %r12b
11915        incl	%ecx
11916        cmpl	%eax, %ecx
11917        jne	L_AES_GCM_decrypt_final_avx1_cmp_tag_loop
11918        cmpb	$0x00, %r12b
11919        sete	%r12b
11920        addq	$16, %rsp
11921        xorq	%rcx, %rcx
11922        jmp	L_AES_GCM_decrypt_final_avx1_cmp_tag_done
11923L_AES_GCM_decrypt_final_avx1_cmp_tag_16:
11924        vmovdqu	(%rsi), %xmm1
11925        vpcmpeqb	%xmm1, %xmm0, %xmm0
11926        vpmovmskb	%xmm0, %rdx
11927        # %%edx == 0xFFFF then return 1 else => return 0
11928        xorl	%r12d, %r12d
11929        cmpl	$0xffff, %edx
11930        sete	%r12b
11931L_AES_GCM_decrypt_final_avx1_cmp_tag_done:
11932        movl	%r12d, (%rbp)
11933        vzeroupper
11934        addq	$16, %rsp
11935        popq	%r12
11936        popq	%rbp
11937        popq	%r13
11938        repz retq
11939#ifndef __APPLE__
11940.size	AES_GCM_decrypt_final_avx1,.-AES_GCM_decrypt_final_avx1
11941#endif /* __APPLE__ */
11942#endif /* WOLFSSL_AESGCM_STREAM */
11943#endif /* HAVE_INTEL_AVX1 */
11944#ifdef HAVE_INTEL_AVX2
11945#ifndef __APPLE__
11946.data
11947#else
11948.section	__DATA,__data
11949#endif /* __APPLE__ */
11950#ifndef __APPLE__
11951.align	16
11952#else
11953.p2align	4
11954#endif /* __APPLE__ */
11955L_GCM_generate_m0_avx2_rev8:
11956.quad	0x08090a0b0c0d0e0f,0x0001020304050607
11957#ifndef __APPLE__
11958.data
11959#else
11960.section	__DATA,__data
11961#endif /* __APPLE__ */
11962#ifndef __APPLE__
11963.align	16
11964#else
11965.p2align	4
11966#endif /* __APPLE__ */
11967L_GCM_generate_m0_avx2_mod2_128:
11968.quad	0x0000000000000000,0xe100000000000000
11969#ifndef __APPLE__
11970.text
11971.globl	GCM_generate_m0_avx2
11972.type	GCM_generate_m0_avx2,@function
11973.align	16
11974GCM_generate_m0_avx2:
11975#else
11976.section	__TEXT,__text
11977.globl	_GCM_generate_m0_avx2
11978.p2align	4
11979_GCM_generate_m0_avx2:
11980#endif /* __APPLE__ */
11981        vmovdqu	L_GCM_generate_m0_avx2_rev8(%rip), %xmm9
11982        vmovdqu	L_GCM_generate_m0_avx2_mod2_128(%rip), %xmm10
11983        vpxor	%xmm8, %xmm8, %xmm8
11984        vmovdqu	(%rdi), %xmm0
11985        vmovdqu	%xmm8, (%rsi)
11986        vmovdqu	%xmm0, %xmm8
11987        vpshufb	%xmm9, %xmm0, %xmm0
11988        vpsllq	$63, %xmm0, %xmm5
11989        vpsrlq	$0x01, %xmm0, %xmm4
11990        vpslldq	$8, %xmm5, %xmm1
11991        vpsrldq	$8, %xmm5, %xmm5
11992        vpshufd	$0xff, %xmm1, %xmm1
11993        vpor	%xmm5, %xmm4, %xmm4
11994        vpsrad	$31, %xmm1, %xmm1
11995        vpand	%xmm10, %xmm1, %xmm1
11996        vpxor	%xmm4, %xmm1, %xmm1
11997        vpsllq	$63, %xmm1, %xmm5
11998        vpsrlq	$0x01, %xmm1, %xmm4
11999        vpslldq	$8, %xmm5, %xmm2
12000        vpsrldq	$8, %xmm5, %xmm5
12001        vpshufd	$0xff, %xmm2, %xmm2
12002        vpor	%xmm5, %xmm4, %xmm4
12003        vpsrad	$31, %xmm2, %xmm2
12004        vpand	%xmm10, %xmm2, %xmm2
12005        vpxor	%xmm4, %xmm2, %xmm2
12006        vpsllq	$63, %xmm2, %xmm5
12007        vpsrlq	$0x01, %xmm2, %xmm4
12008        vpslldq	$8, %xmm5, %xmm3
12009        vpsrldq	$8, %xmm5, %xmm5
12010        vpshufd	$0xff, %xmm3, %xmm3
12011        vpor	%xmm5, %xmm4, %xmm4
12012        vpsrad	$31, %xmm3, %xmm3
12013        vpand	%xmm10, %xmm3, %xmm3
12014        vpxor	%xmm4, %xmm3, %xmm3
12015        vpshufb	%xmm9, %xmm2, %xmm2
12016        vpshufb	%xmm9, %xmm3, %xmm3
12017        vpshufb	%xmm9, %xmm0, %xmm0
12018        vpshufb	%xmm9, %xmm1, %xmm1
12019        vpxor	%xmm2, %xmm3, %xmm8
12020        vmovdqu	%xmm3, 16(%rsi)
12021        vmovdqu	%xmm2, 32(%rsi)
12022        vmovdqu	%xmm8, 48(%rsi)
12023        vmovdqu	%xmm1, 64(%rsi)
12024        vpxor	%xmm1, %xmm3, %xmm4
12025        vpxor	%xmm1, %xmm2, %xmm5
12026        vpxor	%xmm1, %xmm8, %xmm6
12027        vmovdqu	%xmm4, 80(%rsi)
12028        vmovdqu	%xmm5, 96(%rsi)
12029        vmovdqu	%xmm6, 112(%rsi)
12030        vmovdqu	%xmm0, 128(%rsi)
12031        vpxor	%xmm0, %xmm1, %xmm1
12032        vpxor	%xmm0, %xmm3, %xmm4
12033        vpxor	%xmm0, %xmm2, %xmm6
12034        vmovdqu	%xmm4, 144(%rsi)
12035        vmovdqu	%xmm6, 160(%rsi)
12036        vpxor	%xmm6, %xmm3, %xmm6
12037        vmovdqu	%xmm6, 176(%rsi)
12038        vmovdqu	%xmm1, 192(%rsi)
12039        vpxor	%xmm1, %xmm3, %xmm4
12040        vpxor	%xmm1, %xmm2, %xmm5
12041        vpxor	%xmm1, %xmm8, %xmm6
12042        vmovdqu	%xmm4, 208(%rsi)
12043        vmovdqu	%xmm5, 224(%rsi)
12044        vmovdqu	%xmm6, 240(%rsi)
12045        vmovdqu	(%rsi), %xmm0
12046        vmovdqu	16(%rsi), %xmm1
12047        vmovdqu	32(%rsi), %xmm2
12048        vmovdqu	48(%rsi), %xmm3
12049        vpshufb	%xmm9, %xmm0, %xmm0
12050        vpshufb	%xmm9, %xmm1, %xmm1
12051        vpshufb	%xmm9, %xmm2, %xmm2
12052        vpshufb	%xmm9, %xmm3, %xmm3
12053        vpsllq	$60, %xmm0, %xmm4
12054        vpsllq	$60, %xmm1, %xmm5
12055        vpsllq	$60, %xmm2, %xmm6
12056        vpsllq	$60, %xmm3, %xmm7
12057        vpsrlq	$4, %xmm0, %xmm0
12058        vpsrlq	$4, %xmm1, %xmm1
12059        vpsrlq	$4, %xmm2, %xmm2
12060        vpsrlq	$4, %xmm3, %xmm3
12061        vpsrldq	$8, %xmm4, %xmm4
12062        vpsrldq	$8, %xmm5, %xmm5
12063        vpsrldq	$8, %xmm6, %xmm6
12064        vpsrldq	$8, %xmm7, %xmm7
12065        vpor	%xmm4, %xmm0, %xmm0
12066        vpor	%xmm5, %xmm1, %xmm1
12067        vpor	%xmm6, %xmm2, %xmm2
12068        vpor	%xmm7, %xmm3, %xmm3
12069        vpshufb	%xmm9, %xmm0, %xmm0
12070        vpshufb	%xmm9, %xmm1, %xmm1
12071        vpshufb	%xmm9, %xmm2, %xmm2
12072        vpshufb	%xmm9, %xmm3, %xmm3
12073        vmovdqu	%xmm0, 256(%rsi)
12074        vmovdqu	%xmm1, 272(%rsi)
12075        vmovdqu	%xmm2, 288(%rsi)
12076        vmovdqu	%xmm3, 304(%rsi)
12077        vmovdqu	64(%rsi), %xmm0
12078        vmovdqu	80(%rsi), %xmm1
12079        vmovdqu	96(%rsi), %xmm2
12080        vmovdqu	112(%rsi), %xmm3
12081        vpshufb	%xmm9, %xmm0, %xmm0
12082        vpshufb	%xmm9, %xmm1, %xmm1
12083        vpshufb	%xmm9, %xmm2, %xmm2
12084        vpshufb	%xmm9, %xmm3, %xmm3
12085        vpsllq	$60, %xmm0, %xmm4
12086        vpsllq	$60, %xmm1, %xmm5
12087        vpsllq	$60, %xmm2, %xmm6
12088        vpsllq	$60, %xmm3, %xmm7
12089        vpsrlq	$4, %xmm0, %xmm0
12090        vpsrlq	$4, %xmm1, %xmm1
12091        vpsrlq	$4, %xmm2, %xmm2
12092        vpsrlq	$4, %xmm3, %xmm3
12093        vpsrldq	$8, %xmm4, %xmm4
12094        vpsrldq	$8, %xmm5, %xmm5
12095        vpsrldq	$8, %xmm6, %xmm6
12096        vpsrldq	$8, %xmm7, %xmm7
12097        vpor	%xmm4, %xmm0, %xmm0
12098        vpor	%xmm5, %xmm1, %xmm1
12099        vpor	%xmm6, %xmm2, %xmm2
12100        vpor	%xmm7, %xmm3, %xmm3
12101        vpshufb	%xmm9, %xmm0, %xmm0
12102        vpshufb	%xmm9, %xmm1, %xmm1
12103        vpshufb	%xmm9, %xmm2, %xmm2
12104        vpshufb	%xmm9, %xmm3, %xmm3
12105        vmovdqu	%xmm0, 320(%rsi)
12106        vmovdqu	%xmm1, 336(%rsi)
12107        vmovdqu	%xmm2, 352(%rsi)
12108        vmovdqu	%xmm3, 368(%rsi)
12109        vmovdqu	128(%rsi), %xmm0
12110        vmovdqu	144(%rsi), %xmm1
12111        vmovdqu	160(%rsi), %xmm2
12112        vmovdqu	176(%rsi), %xmm3
12113        vpshufb	%xmm9, %xmm0, %xmm0
12114        vpshufb	%xmm9, %xmm1, %xmm1
12115        vpshufb	%xmm9, %xmm2, %xmm2
12116        vpshufb	%xmm9, %xmm3, %xmm3
12117        vpsllq	$60, %xmm0, %xmm4
12118        vpsllq	$60, %xmm1, %xmm5
12119        vpsllq	$60, %xmm2, %xmm6
12120        vpsllq	$60, %xmm3, %xmm7
12121        vpsrlq	$4, %xmm0, %xmm0
12122        vpsrlq	$4, %xmm1, %xmm1
12123        vpsrlq	$4, %xmm2, %xmm2
12124        vpsrlq	$4, %xmm3, %xmm3
12125        vpsrldq	$8, %xmm4, %xmm4
12126        vpsrldq	$8, %xmm5, %xmm5
12127        vpsrldq	$8, %xmm6, %xmm6
12128        vpsrldq	$8, %xmm7, %xmm7
12129        vpor	%xmm4, %xmm0, %xmm0
12130        vpor	%xmm5, %xmm1, %xmm1
12131        vpor	%xmm6, %xmm2, %xmm2
12132        vpor	%xmm7, %xmm3, %xmm3
12133        vpshufb	%xmm9, %xmm0, %xmm0
12134        vpshufb	%xmm9, %xmm1, %xmm1
12135        vpshufb	%xmm9, %xmm2, %xmm2
12136        vpshufb	%xmm9, %xmm3, %xmm3
12137        vmovdqu	%xmm0, 384(%rsi)
12138        vmovdqu	%xmm1, 400(%rsi)
12139        vmovdqu	%xmm2, 416(%rsi)
12140        vmovdqu	%xmm3, 432(%rsi)
12141        vmovdqu	192(%rsi), %xmm0
12142        vmovdqu	208(%rsi), %xmm1
12143        vmovdqu	224(%rsi), %xmm2
12144        vmovdqu	240(%rsi), %xmm3
12145        vpshufb	%xmm9, %xmm0, %xmm0
12146        vpshufb	%xmm9, %xmm1, %xmm1
12147        vpshufb	%xmm9, %xmm2, %xmm2
12148        vpshufb	%xmm9, %xmm3, %xmm3
12149        vpsllq	$60, %xmm0, %xmm4
12150        vpsllq	$60, %xmm1, %xmm5
12151        vpsllq	$60, %xmm2, %xmm6
12152        vpsllq	$60, %xmm3, %xmm7
12153        vpsrlq	$4, %xmm0, %xmm0
12154        vpsrlq	$4, %xmm1, %xmm1
12155        vpsrlq	$4, %xmm2, %xmm2
12156        vpsrlq	$4, %xmm3, %xmm3
12157        vpsrldq	$8, %xmm4, %xmm4
12158        vpsrldq	$8, %xmm5, %xmm5
12159        vpsrldq	$8, %xmm6, %xmm6
12160        vpsrldq	$8, %xmm7, %xmm7
12161        vpor	%xmm4, %xmm0, %xmm0
12162        vpor	%xmm5, %xmm1, %xmm1
12163        vpor	%xmm6, %xmm2, %xmm2
12164        vpor	%xmm7, %xmm3, %xmm3
12165        vpshufb	%xmm9, %xmm0, %xmm0
12166        vpshufb	%xmm9, %xmm1, %xmm1
12167        vpshufb	%xmm9, %xmm2, %xmm2
12168        vpshufb	%xmm9, %xmm3, %xmm3
12169        vmovdqu	%xmm0, 448(%rsi)
12170        vmovdqu	%xmm1, 464(%rsi)
12171        vmovdqu	%xmm2, 480(%rsi)
12172        vmovdqu	%xmm3, 496(%rsi)
12173        repz retq
12174#ifndef __APPLE__
12175.size	GCM_generate_m0_avx2,.-GCM_generate_m0_avx2
12176#endif /* __APPLE__ */
12177#ifndef __APPLE__
12178.data
12179#else
12180.section	__DATA,__data
12181#endif /* __APPLE__ */
12182#ifndef __APPLE__
12183.align	16
12184#else
12185.p2align	4
12186#endif /* __APPLE__ */
12187L_avx2_aes_gcm_one:
12188.quad	0x0000000000000000,0x0000000000000001
12189#ifndef __APPLE__
12190.data
12191#else
12192.section	__DATA,__data
12193#endif /* __APPLE__ */
12194#ifndef __APPLE__
12195.align	16
12196#else
12197.p2align	4
12198#endif /* __APPLE__ */
12199L_avx2_aes_gcm_two:
12200.quad	0x0000000000000000,0x0000000000000002
12201#ifndef __APPLE__
12202.data
12203#else
12204.section	__DATA,__data
12205#endif /* __APPLE__ */
12206#ifndef __APPLE__
12207.align	16
12208#else
12209.p2align	4
12210#endif /* __APPLE__ */
12211L_avx2_aes_gcm_three:
12212.quad	0x0000000000000000,0x0000000000000003
12213#ifndef __APPLE__
12214.data
12215#else
12216.section	__DATA,__data
12217#endif /* __APPLE__ */
12218#ifndef __APPLE__
12219.align	16
12220#else
12221.p2align	4
12222#endif /* __APPLE__ */
12223L_avx2_aes_gcm_four:
12224.quad	0x0000000000000000,0x0000000000000004
12225#ifndef __APPLE__
12226.data
12227#else
12228.section	__DATA,__data
12229#endif /* __APPLE__ */
12230#ifndef __APPLE__
12231.align	16
12232#else
12233.p2align	4
12234#endif /* __APPLE__ */
12235L_avx2_aes_gcm_five:
12236.quad	0x0000000000000000,0x0000000000000005
12237#ifndef __APPLE__
12238.data
12239#else
12240.section	__DATA,__data
12241#endif /* __APPLE__ */
12242#ifndef __APPLE__
12243.align	16
12244#else
12245.p2align	4
12246#endif /* __APPLE__ */
12247L_avx2_aes_gcm_six:
12248.quad	0x0000000000000000,0x0000000000000006
12249#ifndef __APPLE__
12250.data
12251#else
12252.section	__DATA,__data
12253#endif /* __APPLE__ */
12254#ifndef __APPLE__
12255.align	16
12256#else
12257.p2align	4
12258#endif /* __APPLE__ */
12259L_avx2_aes_gcm_seven:
12260.quad	0x0000000000000000,0x0000000000000007
12261#ifndef __APPLE__
12262.data
12263#else
12264.section	__DATA,__data
12265#endif /* __APPLE__ */
12266#ifndef __APPLE__
12267.align	16
12268#else
12269.p2align	4
12270#endif /* __APPLE__ */
12271L_avx2_aes_gcm_eight:
12272.quad	0x0000000000000000,0x0000000000000008
12273#ifndef __APPLE__
12274.data
12275#else
12276.section	__DATA,__data
12277#endif /* __APPLE__ */
12278#ifndef __APPLE__
12279.align	16
12280#else
12281.p2align	4
12282#endif /* __APPLE__ */
12283L_avx2_aes_gcm_bswap_one:
12284.quad	0x0000000000000000,0x0100000000000000
12285#ifndef __APPLE__
12286.data
12287#else
12288.section	__DATA,__data
12289#endif /* __APPLE__ */
12290#ifndef __APPLE__
12291.align	16
12292#else
12293.p2align	4
12294#endif /* __APPLE__ */
12295L_avx2_aes_gcm_bswap_epi64:
12296.quad	0x0001020304050607,0x08090a0b0c0d0e0f
12297#ifndef __APPLE__
12298.data
12299#else
12300.section	__DATA,__data
12301#endif /* __APPLE__ */
12302#ifndef __APPLE__
12303.align	16
12304#else
12305.p2align	4
12306#endif /* __APPLE__ */
12307L_avx2_aes_gcm_bswap_mask:
12308.quad	0x08090a0b0c0d0e0f,0x0001020304050607
12309#ifndef __APPLE__
12310.data
12311#else
12312.section	__DATA,__data
12313#endif /* __APPLE__ */
12314#ifndef __APPLE__
12315.align	16
12316#else
12317.p2align	4
12318#endif /* __APPLE__ */
12319L_avx2_aes_gcm_mod2_128:
12320.quad	0x0000000000000001,0xc200000000000000
12321#ifndef __APPLE__
12322.text
12323.globl	AES_GCM_encrypt_avx2
12324.type	AES_GCM_encrypt_avx2,@function
12325.align	16
12326AES_GCM_encrypt_avx2:
12327#else
12328.section	__TEXT,__text
12329.globl	_AES_GCM_encrypt_avx2
12330.p2align	4
12331_AES_GCM_encrypt_avx2:
12332#endif /* __APPLE__ */
12333        pushq	%r13
12334        pushq	%r12
12335        pushq	%r15
12336        pushq	%rbx
12337        pushq	%r14
12338        movq	%rdx, %r12
12339        movq	%rcx, %rax
12340        movq	%r8, %r15
12341        movq	%rsi, %r8
12342        movl	%r9d, %r10d
12343        movl	48(%rsp), %r11d
12344        movl	56(%rsp), %ebx
12345        movl	64(%rsp), %r14d
12346        movq	72(%rsp), %rsi
12347        movl	80(%rsp), %r9d
12348        subq	$0xa0, %rsp
12349        vpxor	%xmm4, %xmm4, %xmm4
12350        vpxor	%xmm6, %xmm6, %xmm6
12351        movl	%ebx, %edx
12352        cmpl	$12, %edx
12353        je	L_AES_GCM_encrypt_avx2_iv_12
12354        # Calculate values when IV is not 12 bytes
12355        # H = Encrypt X(=0)
12356        vmovdqu	(%rsi), %xmm5
12357        vaesenc	16(%rsi), %xmm5, %xmm5
12358        vaesenc	32(%rsi), %xmm5, %xmm5
12359        vaesenc	48(%rsi), %xmm5, %xmm5
12360        vaesenc	64(%rsi), %xmm5, %xmm5
12361        vaesenc	80(%rsi), %xmm5, %xmm5
12362        vaesenc	96(%rsi), %xmm5, %xmm5
12363        vaesenc	112(%rsi), %xmm5, %xmm5
12364        vaesenc	128(%rsi), %xmm5, %xmm5
12365        vaesenc	144(%rsi), %xmm5, %xmm5
12366        cmpl	$11, %r9d
12367        vmovdqu	160(%rsi), %xmm0
12368        jl	L_AES_GCM_encrypt_avx2_calc_iv_1_aesenc_avx_last
12369        vaesenc	%xmm0, %xmm5, %xmm5
12370        vaesenc	176(%rsi), %xmm5, %xmm5
12371        cmpl	$13, %r9d
12372        vmovdqu	192(%rsi), %xmm0
12373        jl	L_AES_GCM_encrypt_avx2_calc_iv_1_aesenc_avx_last
12374        vaesenc	%xmm0, %xmm5, %xmm5
12375        vaesenc	208(%rsi), %xmm5, %xmm5
12376        vmovdqu	224(%rsi), %xmm0
12377L_AES_GCM_encrypt_avx2_calc_iv_1_aesenc_avx_last:
12378        vaesenclast	%xmm0, %xmm5, %xmm5
12379        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm5, %xmm5
12380        # Calc counter
12381        # Initialization vector
12382        cmpl	$0x00, %edx
12383        movq	$0x00, %rcx
12384        je	L_AES_GCM_encrypt_avx2_calc_iv_done
12385        cmpl	$16, %edx
12386        jl	L_AES_GCM_encrypt_avx2_calc_iv_lt16
12387        andl	$0xfffffff0, %edx
12388L_AES_GCM_encrypt_avx2_calc_iv_16_loop:
12389        vmovdqu	(%rax,%rcx,1), %xmm0
12390        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0
12391        vpxor	%xmm0, %xmm4, %xmm4
12392        # ghash_gfmul_avx
12393        vpclmulqdq	$16, %xmm4, %xmm5, %xmm2
12394        vpclmulqdq	$0x01, %xmm4, %xmm5, %xmm1
12395        vpclmulqdq	$0x00, %xmm4, %xmm5, %xmm0
12396        vpclmulqdq	$0x11, %xmm4, %xmm5, %xmm3
12397        vpxor	%xmm1, %xmm2, %xmm2
12398        vpslldq	$8, %xmm2, %xmm1
12399        vpsrldq	$8, %xmm2, %xmm2
12400        vpxor	%xmm1, %xmm0, %xmm7
12401        vpxor	%xmm2, %xmm3, %xmm4
12402        # ghash_mid
12403        vpsrld	$31, %xmm7, %xmm0
12404        vpsrld	$31, %xmm4, %xmm1
12405        vpslld	$0x01, %xmm7, %xmm7
12406        vpslld	$0x01, %xmm4, %xmm4
12407        vpsrldq	$12, %xmm0, %xmm2
12408        vpslldq	$4, %xmm0, %xmm0
12409        vpslldq	$4, %xmm1, %xmm1
12410        vpor	%xmm2, %xmm4, %xmm4
12411        vpor	%xmm0, %xmm7, %xmm7
12412        vpor	%xmm1, %xmm4, %xmm4
12413        # ghash_red
12414        vmovdqu	L_avx2_aes_gcm_mod2_128(%rip), %xmm2
12415        vpclmulqdq	$16, %xmm2, %xmm7, %xmm0
12416        vpshufd	$0x4e, %xmm7, %xmm1
12417        vpxor	%xmm0, %xmm1, %xmm1
12418        vpclmulqdq	$16, %xmm2, %xmm1, %xmm0
12419        vpshufd	$0x4e, %xmm1, %xmm1
12420        vpxor	%xmm0, %xmm1, %xmm1
12421        vpxor	%xmm1, %xmm4, %xmm4
12422        addl	$16, %ecx
12423        cmpl	%edx, %ecx
12424        jl	L_AES_GCM_encrypt_avx2_calc_iv_16_loop
12425        movl	%ebx, %edx
12426        cmpl	%edx, %ecx
12427        je	L_AES_GCM_encrypt_avx2_calc_iv_done
12428L_AES_GCM_encrypt_avx2_calc_iv_lt16:
12429        vpxor	%xmm0, %xmm0, %xmm0
12430        xorl	%ebx, %ebx
12431        vmovdqu	%xmm0, (%rsp)
12432L_AES_GCM_encrypt_avx2_calc_iv_loop:
12433        movzbl	(%rax,%rcx,1), %r13d
12434        movb	%r13b, (%rsp,%rbx,1)
12435        incl	%ecx
12436        incl	%ebx
12437        cmpl	%edx, %ecx
12438        jl	L_AES_GCM_encrypt_avx2_calc_iv_loop
12439        vmovdqu	(%rsp), %xmm0
12440        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0
12441        vpxor	%xmm0, %xmm4, %xmm4
12442        # ghash_gfmul_avx
12443        vpclmulqdq	$16, %xmm4, %xmm5, %xmm2
12444        vpclmulqdq	$0x01, %xmm4, %xmm5, %xmm1
12445        vpclmulqdq	$0x00, %xmm4, %xmm5, %xmm0
12446        vpclmulqdq	$0x11, %xmm4, %xmm5, %xmm3
12447        vpxor	%xmm1, %xmm2, %xmm2
12448        vpslldq	$8, %xmm2, %xmm1
12449        vpsrldq	$8, %xmm2, %xmm2
12450        vpxor	%xmm1, %xmm0, %xmm7
12451        vpxor	%xmm2, %xmm3, %xmm4
12452        # ghash_mid
12453        vpsrld	$31, %xmm7, %xmm0
12454        vpsrld	$31, %xmm4, %xmm1
12455        vpslld	$0x01, %xmm7, %xmm7
12456        vpslld	$0x01, %xmm4, %xmm4
12457        vpsrldq	$12, %xmm0, %xmm2
12458        vpslldq	$4, %xmm0, %xmm0
12459        vpslldq	$4, %xmm1, %xmm1
12460        vpor	%xmm2, %xmm4, %xmm4
12461        vpor	%xmm0, %xmm7, %xmm7
12462        vpor	%xmm1, %xmm4, %xmm4
12463        # ghash_red
12464        vmovdqu	L_avx2_aes_gcm_mod2_128(%rip), %xmm2
12465        vpclmulqdq	$16, %xmm2, %xmm7, %xmm0
12466        vpshufd	$0x4e, %xmm7, %xmm1
12467        vpxor	%xmm0, %xmm1, %xmm1
12468        vpclmulqdq	$16, %xmm2, %xmm1, %xmm0
12469        vpshufd	$0x4e, %xmm1, %xmm1
12470        vpxor	%xmm0, %xmm1, %xmm1
12471        vpxor	%xmm1, %xmm4, %xmm4
12472L_AES_GCM_encrypt_avx2_calc_iv_done:
12473        # T = Encrypt counter
12474        vpxor	%xmm0, %xmm0, %xmm0
12475        shll	$3, %edx
12476        vmovq	%rdx, %xmm0
12477        vpxor	%xmm0, %xmm4, %xmm4
12478        # ghash_gfmul_avx
12479        vpclmulqdq	$16, %xmm4, %xmm5, %xmm2
12480        vpclmulqdq	$0x01, %xmm4, %xmm5, %xmm1
12481        vpclmulqdq	$0x00, %xmm4, %xmm5, %xmm0
12482        vpclmulqdq	$0x11, %xmm4, %xmm5, %xmm3
12483        vpxor	%xmm1, %xmm2, %xmm2
12484        vpslldq	$8, %xmm2, %xmm1
12485        vpsrldq	$8, %xmm2, %xmm2
12486        vpxor	%xmm1, %xmm0, %xmm7
12487        vpxor	%xmm2, %xmm3, %xmm4
12488        # ghash_mid
12489        vpsrld	$31, %xmm7, %xmm0
12490        vpsrld	$31, %xmm4, %xmm1
12491        vpslld	$0x01, %xmm7, %xmm7
12492        vpslld	$0x01, %xmm4, %xmm4
12493        vpsrldq	$12, %xmm0, %xmm2
12494        vpslldq	$4, %xmm0, %xmm0
12495        vpslldq	$4, %xmm1, %xmm1
12496        vpor	%xmm2, %xmm4, %xmm4
12497        vpor	%xmm0, %xmm7, %xmm7
12498        vpor	%xmm1, %xmm4, %xmm4
12499        # ghash_red
12500        vmovdqu	L_avx2_aes_gcm_mod2_128(%rip), %xmm2
12501        vpclmulqdq	$16, %xmm2, %xmm7, %xmm0
12502        vpshufd	$0x4e, %xmm7, %xmm1
12503        vpxor	%xmm0, %xmm1, %xmm1
12504        vpclmulqdq	$16, %xmm2, %xmm1, %xmm0
12505        vpshufd	$0x4e, %xmm1, %xmm1
12506        vpxor	%xmm0, %xmm1, %xmm1
12507        vpxor	%xmm1, %xmm4, %xmm4
12508        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm4, %xmm4
12509        #   Encrypt counter
12510        vmovdqu	(%rsi), %xmm15
12511        vpxor	%xmm4, %xmm15, %xmm15
12512        vaesenc	16(%rsi), %xmm15, %xmm15
12513        vaesenc	32(%rsi), %xmm15, %xmm15
12514        vaesenc	48(%rsi), %xmm15, %xmm15
12515        vaesenc	64(%rsi), %xmm15, %xmm15
12516        vaesenc	80(%rsi), %xmm15, %xmm15
12517        vaesenc	96(%rsi), %xmm15, %xmm15
12518        vaesenc	112(%rsi), %xmm15, %xmm15
12519        vaesenc	128(%rsi), %xmm15, %xmm15
12520        vaesenc	144(%rsi), %xmm15, %xmm15
12521        cmpl	$11, %r9d
12522        vmovdqu	160(%rsi), %xmm0
12523        jl	L_AES_GCM_encrypt_avx2_calc_iv_2_aesenc_avx_last
12524        vaesenc	%xmm0, %xmm15, %xmm15
12525        vaesenc	176(%rsi), %xmm15, %xmm15
12526        cmpl	$13, %r9d
12527        vmovdqu	192(%rsi), %xmm0
12528        jl	L_AES_GCM_encrypt_avx2_calc_iv_2_aesenc_avx_last
12529        vaesenc	%xmm0, %xmm15, %xmm15
12530        vaesenc	208(%rsi), %xmm15, %xmm15
12531        vmovdqu	224(%rsi), %xmm0
12532L_AES_GCM_encrypt_avx2_calc_iv_2_aesenc_avx_last:
12533        vaesenclast	%xmm0, %xmm15, %xmm15
12534        jmp	L_AES_GCM_encrypt_avx2_iv_done
12535L_AES_GCM_encrypt_avx2_iv_12:
12536        # # Calculate values when IV is 12 bytes
12537        # Set counter based on IV
12538        vmovdqu	L_avx2_aes_gcm_bswap_one(%rip), %xmm4
12539        vmovdqu	(%rsi), %xmm5
12540        vpblendd	$7, (%rax), %xmm4, %xmm4
12541        # H = Encrypt X(=0) and T = Encrypt counter
12542        vmovdqu	16(%rsi), %xmm7
12543        vpxor	%xmm5, %xmm4, %xmm15
12544        vaesenc	%xmm7, %xmm5, %xmm5
12545        vaesenc	%xmm7, %xmm15, %xmm15
12546        vmovdqu	32(%rsi), %xmm0
12547        vaesenc	%xmm0, %xmm5, %xmm5
12548        vaesenc	%xmm0, %xmm15, %xmm15
12549        vmovdqu	48(%rsi), %xmm0
12550        vaesenc	%xmm0, %xmm5, %xmm5
12551        vaesenc	%xmm0, %xmm15, %xmm15
12552        vmovdqu	64(%rsi), %xmm0
12553        vaesenc	%xmm0, %xmm5, %xmm5
12554        vaesenc	%xmm0, %xmm15, %xmm15
12555        vmovdqu	80(%rsi), %xmm0
12556        vaesenc	%xmm0, %xmm5, %xmm5
12557        vaesenc	%xmm0, %xmm15, %xmm15
12558        vmovdqu	96(%rsi), %xmm0
12559        vaesenc	%xmm0, %xmm5, %xmm5
12560        vaesenc	%xmm0, %xmm15, %xmm15
12561        vmovdqu	112(%rsi), %xmm0
12562        vaesenc	%xmm0, %xmm5, %xmm5
12563        vaesenc	%xmm0, %xmm15, %xmm15
12564        vmovdqu	128(%rsi), %xmm0
12565        vaesenc	%xmm0, %xmm5, %xmm5
12566        vaesenc	%xmm0, %xmm15, %xmm15
12567        vmovdqu	144(%rsi), %xmm0
12568        vaesenc	%xmm0, %xmm5, %xmm5
12569        vaesenc	%xmm0, %xmm15, %xmm15
12570        cmpl	$11, %r9d
12571        vmovdqu	160(%rsi), %xmm0
12572        jl	L_AES_GCM_encrypt_avx2_calc_iv_12_last
12573        vaesenc	%xmm0, %xmm5, %xmm5
12574        vaesenc	%xmm0, %xmm15, %xmm15
12575        vmovdqu	176(%rsi), %xmm0
12576        vaesenc	%xmm0, %xmm5, %xmm5
12577        vaesenc	%xmm0, %xmm15, %xmm15
12578        cmpl	$13, %r9d
12579        vmovdqu	192(%rsi), %xmm0
12580        jl	L_AES_GCM_encrypt_avx2_calc_iv_12_last
12581        vaesenc	%xmm0, %xmm5, %xmm5
12582        vaesenc	%xmm0, %xmm15, %xmm15
12583        vmovdqu	208(%rsi), %xmm0
12584        vaesenc	%xmm0, %xmm5, %xmm5
12585        vaesenc	%xmm0, %xmm15, %xmm15
12586        vmovdqu	224(%rsi), %xmm0
12587L_AES_GCM_encrypt_avx2_calc_iv_12_last:
12588        vaesenclast	%xmm0, %xmm5, %xmm5
12589        vaesenclast	%xmm0, %xmm15, %xmm15
12590        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm5, %xmm5
12591L_AES_GCM_encrypt_avx2_iv_done:
12592        # Additional authentication data
12593        movl	%r11d, %edx
12594        cmpl	$0x00, %edx
12595        je	L_AES_GCM_encrypt_avx2_calc_aad_done
12596        xorl	%ecx, %ecx
12597        cmpl	$16, %edx
12598        jl	L_AES_GCM_encrypt_avx2_calc_aad_lt16
12599        andl	$0xfffffff0, %edx
12600L_AES_GCM_encrypt_avx2_calc_aad_16_loop:
12601        vmovdqu	(%r12,%rcx,1), %xmm0
12602        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0
12603        vpxor	%xmm0, %xmm6, %xmm6
12604        # ghash_gfmul_avx
12605        vpclmulqdq	$16, %xmm6, %xmm5, %xmm2
12606        vpclmulqdq	$0x01, %xmm6, %xmm5, %xmm1
12607        vpclmulqdq	$0x00, %xmm6, %xmm5, %xmm0
12608        vpclmulqdq	$0x11, %xmm6, %xmm5, %xmm3
12609        vpxor	%xmm1, %xmm2, %xmm2
12610        vpslldq	$8, %xmm2, %xmm1
12611        vpsrldq	$8, %xmm2, %xmm2
12612        vpxor	%xmm1, %xmm0, %xmm7
12613        vpxor	%xmm2, %xmm3, %xmm6
12614        # ghash_mid
12615        vpsrld	$31, %xmm7, %xmm0
12616        vpsrld	$31, %xmm6, %xmm1
12617        vpslld	$0x01, %xmm7, %xmm7
12618        vpslld	$0x01, %xmm6, %xmm6
12619        vpsrldq	$12, %xmm0, %xmm2
12620        vpslldq	$4, %xmm0, %xmm0
12621        vpslldq	$4, %xmm1, %xmm1
12622        vpor	%xmm2, %xmm6, %xmm6
12623        vpor	%xmm0, %xmm7, %xmm7
12624        vpor	%xmm1, %xmm6, %xmm6
12625        # ghash_red
12626        vmovdqu	L_avx2_aes_gcm_mod2_128(%rip), %xmm2
12627        vpclmulqdq	$16, %xmm2, %xmm7, %xmm0
12628        vpshufd	$0x4e, %xmm7, %xmm1
12629        vpxor	%xmm0, %xmm1, %xmm1
12630        vpclmulqdq	$16, %xmm2, %xmm1, %xmm0
12631        vpshufd	$0x4e, %xmm1, %xmm1
12632        vpxor	%xmm0, %xmm1, %xmm1
12633        vpxor	%xmm1, %xmm6, %xmm6
12634        addl	$16, %ecx
12635        cmpl	%edx, %ecx
12636        jl	L_AES_GCM_encrypt_avx2_calc_aad_16_loop
12637        movl	%r11d, %edx
12638        cmpl	%edx, %ecx
12639        je	L_AES_GCM_encrypt_avx2_calc_aad_done
12640L_AES_GCM_encrypt_avx2_calc_aad_lt16:
12641        vpxor	%xmm0, %xmm0, %xmm0
12642        xorl	%ebx, %ebx
12643        vmovdqu	%xmm0, (%rsp)
12644L_AES_GCM_encrypt_avx2_calc_aad_loop:
12645        movzbl	(%r12,%rcx,1), %r13d
12646        movb	%r13b, (%rsp,%rbx,1)
12647        incl	%ecx
12648        incl	%ebx
12649        cmpl	%edx, %ecx
12650        jl	L_AES_GCM_encrypt_avx2_calc_aad_loop
12651        vmovdqu	(%rsp), %xmm0
12652        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0
12653        vpxor	%xmm0, %xmm6, %xmm6
12654        # ghash_gfmul_avx
12655        vpclmulqdq	$16, %xmm6, %xmm5, %xmm2
12656        vpclmulqdq	$0x01, %xmm6, %xmm5, %xmm1
12657        vpclmulqdq	$0x00, %xmm6, %xmm5, %xmm0
12658        vpclmulqdq	$0x11, %xmm6, %xmm5, %xmm3
12659        vpxor	%xmm1, %xmm2, %xmm2
12660        vpslldq	$8, %xmm2, %xmm1
12661        vpsrldq	$8, %xmm2, %xmm2
12662        vpxor	%xmm1, %xmm0, %xmm7
12663        vpxor	%xmm2, %xmm3, %xmm6
12664        # ghash_mid
12665        vpsrld	$31, %xmm7, %xmm0
12666        vpsrld	$31, %xmm6, %xmm1
12667        vpslld	$0x01, %xmm7, %xmm7
12668        vpslld	$0x01, %xmm6, %xmm6
12669        vpsrldq	$12, %xmm0, %xmm2
12670        vpslldq	$4, %xmm0, %xmm0
12671        vpslldq	$4, %xmm1, %xmm1
12672        vpor	%xmm2, %xmm6, %xmm6
12673        vpor	%xmm0, %xmm7, %xmm7
12674        vpor	%xmm1, %xmm6, %xmm6
12675        # ghash_red
12676        vmovdqu	L_avx2_aes_gcm_mod2_128(%rip), %xmm2
12677        vpclmulqdq	$16, %xmm2, %xmm7, %xmm0
12678        vpshufd	$0x4e, %xmm7, %xmm1
12679        vpxor	%xmm0, %xmm1, %xmm1
12680        vpclmulqdq	$16, %xmm2, %xmm1, %xmm0
12681        vpshufd	$0x4e, %xmm1, %xmm1
12682        vpxor	%xmm0, %xmm1, %xmm1
12683        vpxor	%xmm1, %xmm6, %xmm6
12684L_AES_GCM_encrypt_avx2_calc_aad_done:
12685        # Calculate counter and H
12686        vpsrlq	$63, %xmm5, %xmm1
12687        vpsllq	$0x01, %xmm5, %xmm0
12688        vpslldq	$8, %xmm1, %xmm1
12689        vpor	%xmm1, %xmm0, %xmm0
12690        vpshufd	$0xff, %xmm5, %xmm5
12691        vpsrad	$31, %xmm5, %xmm5
12692        vpshufb	L_avx2_aes_gcm_bswap_epi64(%rip), %xmm4, %xmm4
12693        vpand	L_avx2_aes_gcm_mod2_128(%rip), %xmm5, %xmm5
12694        vpaddd	L_avx2_aes_gcm_one(%rip), %xmm4, %xmm4
12695        vpxor	%xmm0, %xmm5, %xmm5
12696        xorl	%ebx, %ebx
12697        cmpl	$0x80, %r10d
12698        movl	%r10d, %r13d
12699        jl	L_AES_GCM_encrypt_avx2_done_128
12700        andl	$0xffffff80, %r13d
12701        vmovdqu	%xmm4, 128(%rsp)
12702        vmovdqu	%xmm15, 144(%rsp)
12703        vmovdqu	L_avx2_aes_gcm_mod2_128(%rip), %xmm3
12704        # H ^ 1 and H ^ 2
12705        vpclmulqdq	$0x00, %xmm5, %xmm5, %xmm9
12706        vpclmulqdq	$0x11, %xmm5, %xmm5, %xmm10
12707        vpclmulqdq	$16, %xmm3, %xmm9, %xmm8
12708        vpshufd	$0x4e, %xmm9, %xmm9
12709        vpxor	%xmm8, %xmm9, %xmm9
12710        vpclmulqdq	$16, %xmm3, %xmm9, %xmm8
12711        vpshufd	$0x4e, %xmm9, %xmm9
12712        vpxor	%xmm8, %xmm9, %xmm9
12713        vpxor	%xmm9, %xmm10, %xmm0
12714        vmovdqu	%xmm5, (%rsp)
12715        vmovdqu	%xmm0, 16(%rsp)
12716        # H ^ 3 and H ^ 4
12717        vpclmulqdq	$16, %xmm5, %xmm0, %xmm11
12718        vpclmulqdq	$0x01, %xmm5, %xmm0, %xmm10
12719        vpclmulqdq	$0x00, %xmm5, %xmm0, %xmm9
12720        vpclmulqdq	$0x11, %xmm5, %xmm0, %xmm12
12721        vpclmulqdq	$0x00, %xmm0, %xmm0, %xmm13
12722        vpclmulqdq	$0x11, %xmm0, %xmm0, %xmm14
12723        vpxor	%xmm10, %xmm11, %xmm11
12724        vpslldq	$8, %xmm11, %xmm10
12725        vpsrldq	$8, %xmm11, %xmm11
12726        vpxor	%xmm9, %xmm10, %xmm10
12727        vpclmulqdq	$16, %xmm3, %xmm13, %xmm8
12728        vpclmulqdq	$16, %xmm3, %xmm10, %xmm9
12729        vpshufd	$0x4e, %xmm10, %xmm10
12730        vpshufd	$0x4e, %xmm13, %xmm13
12731        vpxor	%xmm9, %xmm10, %xmm10
12732        vpxor	%xmm8, %xmm13, %xmm13
12733        vpclmulqdq	$16, %xmm3, %xmm10, %xmm9
12734        vpclmulqdq	$16, %xmm3, %xmm13, %xmm8
12735        vpshufd	$0x4e, %xmm10, %xmm10
12736        vpshufd	$0x4e, %xmm13, %xmm13
12737        vpxor	%xmm11, %xmm12, %xmm12
12738        vpxor	%xmm8, %xmm13, %xmm13
12739        vpxor	%xmm12, %xmm10, %xmm10
12740        vpxor	%xmm14, %xmm13, %xmm2
12741        vpxor	%xmm9, %xmm10, %xmm1
12742        vmovdqu	%xmm1, 32(%rsp)
12743        vmovdqu	%xmm2, 48(%rsp)
12744        # H ^ 5 and H ^ 6
12745        vpclmulqdq	$16, %xmm0, %xmm1, %xmm11
12746        vpclmulqdq	$0x01, %xmm0, %xmm1, %xmm10
12747        vpclmulqdq	$0x00, %xmm0, %xmm1, %xmm9
12748        vpclmulqdq	$0x11, %xmm0, %xmm1, %xmm12
12749        vpclmulqdq	$0x00, %xmm1, %xmm1, %xmm13
12750        vpclmulqdq	$0x11, %xmm1, %xmm1, %xmm14
12751        vpxor	%xmm10, %xmm11, %xmm11
12752        vpslldq	$8, %xmm11, %xmm10
12753        vpsrldq	$8, %xmm11, %xmm11
12754        vpxor	%xmm9, %xmm10, %xmm10
12755        vpclmulqdq	$16, %xmm3, %xmm13, %xmm8
12756        vpclmulqdq	$16, %xmm3, %xmm10, %xmm9
12757        vpshufd	$0x4e, %xmm10, %xmm10
12758        vpshufd	$0x4e, %xmm13, %xmm13
12759        vpxor	%xmm9, %xmm10, %xmm10
12760        vpxor	%xmm8, %xmm13, %xmm13
12761        vpclmulqdq	$16, %xmm3, %xmm10, %xmm9
12762        vpclmulqdq	$16, %xmm3, %xmm13, %xmm8
12763        vpshufd	$0x4e, %xmm10, %xmm10
12764        vpshufd	$0x4e, %xmm13, %xmm13
12765        vpxor	%xmm11, %xmm12, %xmm12
12766        vpxor	%xmm8, %xmm13, %xmm13
12767        vpxor	%xmm12, %xmm10, %xmm10
12768        vpxor	%xmm14, %xmm13, %xmm0
12769        vpxor	%xmm9, %xmm10, %xmm7
12770        vmovdqu	%xmm7, 64(%rsp)
12771        vmovdqu	%xmm0, 80(%rsp)
12772        # H ^ 7 and H ^ 8
12773        vpclmulqdq	$16, %xmm1, %xmm2, %xmm11
12774        vpclmulqdq	$0x01, %xmm1, %xmm2, %xmm10
12775        vpclmulqdq	$0x00, %xmm1, %xmm2, %xmm9
12776        vpclmulqdq	$0x11, %xmm1, %xmm2, %xmm12
12777        vpclmulqdq	$0x00, %xmm2, %xmm2, %xmm13
12778        vpclmulqdq	$0x11, %xmm2, %xmm2, %xmm14
12779        vpxor	%xmm10, %xmm11, %xmm11
12780        vpslldq	$8, %xmm11, %xmm10
12781        vpsrldq	$8, %xmm11, %xmm11
12782        vpxor	%xmm9, %xmm10, %xmm10
12783        vpclmulqdq	$16, %xmm3, %xmm13, %xmm8
12784        vpclmulqdq	$16, %xmm3, %xmm10, %xmm9
12785        vpshufd	$0x4e, %xmm10, %xmm10
12786        vpshufd	$0x4e, %xmm13, %xmm13
12787        vpxor	%xmm9, %xmm10, %xmm10
12788        vpxor	%xmm8, %xmm13, %xmm13
12789        vpclmulqdq	$16, %xmm3, %xmm10, %xmm9
12790        vpclmulqdq	$16, %xmm3, %xmm13, %xmm8
12791        vpshufd	$0x4e, %xmm10, %xmm10
12792        vpshufd	$0x4e, %xmm13, %xmm13
12793        vpxor	%xmm11, %xmm12, %xmm12
12794        vpxor	%xmm8, %xmm13, %xmm13
12795        vpxor	%xmm12, %xmm10, %xmm10
12796        vpxor	%xmm14, %xmm13, %xmm0
12797        vpxor	%xmm9, %xmm10, %xmm7
12798        vmovdqu	%xmm7, 96(%rsp)
12799        vmovdqu	%xmm0, 112(%rsp)
12800        # First 128 bytes of input
12801        # aesenc_128
12802        # aesenc_ctr
12803        vmovdqu	128(%rsp), %xmm0
12804        vmovdqu	L_avx2_aes_gcm_bswap_epi64(%rip), %xmm1
12805        vpaddd	L_avx2_aes_gcm_one(%rip), %xmm0, %xmm9
12806        vpshufb	%xmm1, %xmm0, %xmm8
12807        vpaddd	L_avx2_aes_gcm_two(%rip), %xmm0, %xmm10
12808        vpshufb	%xmm1, %xmm9, %xmm9
12809        vpaddd	L_avx2_aes_gcm_three(%rip), %xmm0, %xmm11
12810        vpshufb	%xmm1, %xmm10, %xmm10
12811        vpaddd	L_avx2_aes_gcm_four(%rip), %xmm0, %xmm12
12812        vpshufb	%xmm1, %xmm11, %xmm11
12813        vpaddd	L_avx2_aes_gcm_five(%rip), %xmm0, %xmm13
12814        vpshufb	%xmm1, %xmm12, %xmm12
12815        vpaddd	L_avx2_aes_gcm_six(%rip), %xmm0, %xmm14
12816        vpshufb	%xmm1, %xmm13, %xmm13
12817        vpaddd	L_avx2_aes_gcm_seven(%rip), %xmm0, %xmm15
12818        vpshufb	%xmm1, %xmm14, %xmm14
12819        vpaddd	L_avx2_aes_gcm_eight(%rip), %xmm0, %xmm0
12820        vpshufb	%xmm1, %xmm15, %xmm15
12821        # aesenc_xor
12822        vmovdqu	(%rsi), %xmm7
12823        vmovdqu	%xmm0, 128(%rsp)
12824        vpxor	%xmm7, %xmm8, %xmm8
12825        vpxor	%xmm7, %xmm9, %xmm9
12826        vpxor	%xmm7, %xmm10, %xmm10
12827        vpxor	%xmm7, %xmm11, %xmm11
12828        vpxor	%xmm7, %xmm12, %xmm12
12829        vpxor	%xmm7, %xmm13, %xmm13
12830        vpxor	%xmm7, %xmm14, %xmm14
12831        vpxor	%xmm7, %xmm15, %xmm15
12832        vmovdqu	16(%rsi), %xmm7
12833        vaesenc	%xmm7, %xmm8, %xmm8
12834        vaesenc	%xmm7, %xmm9, %xmm9
12835        vaesenc	%xmm7, %xmm10, %xmm10
12836        vaesenc	%xmm7, %xmm11, %xmm11
12837        vaesenc	%xmm7, %xmm12, %xmm12
12838        vaesenc	%xmm7, %xmm13, %xmm13
12839        vaesenc	%xmm7, %xmm14, %xmm14
12840        vaesenc	%xmm7, %xmm15, %xmm15
12841        vmovdqu	32(%rsi), %xmm7
12842        vaesenc	%xmm7, %xmm8, %xmm8
12843        vaesenc	%xmm7, %xmm9, %xmm9
12844        vaesenc	%xmm7, %xmm10, %xmm10
12845        vaesenc	%xmm7, %xmm11, %xmm11
12846        vaesenc	%xmm7, %xmm12, %xmm12
12847        vaesenc	%xmm7, %xmm13, %xmm13
12848        vaesenc	%xmm7, %xmm14, %xmm14
12849        vaesenc	%xmm7, %xmm15, %xmm15
12850        vmovdqu	48(%rsi), %xmm7
12851        vaesenc	%xmm7, %xmm8, %xmm8
12852        vaesenc	%xmm7, %xmm9, %xmm9
12853        vaesenc	%xmm7, %xmm10, %xmm10
12854        vaesenc	%xmm7, %xmm11, %xmm11
12855        vaesenc	%xmm7, %xmm12, %xmm12
12856        vaesenc	%xmm7, %xmm13, %xmm13
12857        vaesenc	%xmm7, %xmm14, %xmm14
12858        vaesenc	%xmm7, %xmm15, %xmm15
12859        vmovdqu	64(%rsi), %xmm7
12860        vaesenc	%xmm7, %xmm8, %xmm8
12861        vaesenc	%xmm7, %xmm9, %xmm9
12862        vaesenc	%xmm7, %xmm10, %xmm10
12863        vaesenc	%xmm7, %xmm11, %xmm11
12864        vaesenc	%xmm7, %xmm12, %xmm12
12865        vaesenc	%xmm7, %xmm13, %xmm13
12866        vaesenc	%xmm7, %xmm14, %xmm14
12867        vaesenc	%xmm7, %xmm15, %xmm15
12868        vmovdqu	80(%rsi), %xmm7
12869        vaesenc	%xmm7, %xmm8, %xmm8
12870        vaesenc	%xmm7, %xmm9, %xmm9
12871        vaesenc	%xmm7, %xmm10, %xmm10
12872        vaesenc	%xmm7, %xmm11, %xmm11
12873        vaesenc	%xmm7, %xmm12, %xmm12
12874        vaesenc	%xmm7, %xmm13, %xmm13
12875        vaesenc	%xmm7, %xmm14, %xmm14
12876        vaesenc	%xmm7, %xmm15, %xmm15
12877        vmovdqu	96(%rsi), %xmm7
12878        vaesenc	%xmm7, %xmm8, %xmm8
12879        vaesenc	%xmm7, %xmm9, %xmm9
12880        vaesenc	%xmm7, %xmm10, %xmm10
12881        vaesenc	%xmm7, %xmm11, %xmm11
12882        vaesenc	%xmm7, %xmm12, %xmm12
12883        vaesenc	%xmm7, %xmm13, %xmm13
12884        vaesenc	%xmm7, %xmm14, %xmm14
12885        vaesenc	%xmm7, %xmm15, %xmm15
12886        vmovdqu	112(%rsi), %xmm7
12887        vaesenc	%xmm7, %xmm8, %xmm8
12888        vaesenc	%xmm7, %xmm9, %xmm9
12889        vaesenc	%xmm7, %xmm10, %xmm10
12890        vaesenc	%xmm7, %xmm11, %xmm11
12891        vaesenc	%xmm7, %xmm12, %xmm12
12892        vaesenc	%xmm7, %xmm13, %xmm13
12893        vaesenc	%xmm7, %xmm14, %xmm14
12894        vaesenc	%xmm7, %xmm15, %xmm15
12895        vmovdqu	128(%rsi), %xmm7
12896        vaesenc	%xmm7, %xmm8, %xmm8
12897        vaesenc	%xmm7, %xmm9, %xmm9
12898        vaesenc	%xmm7, %xmm10, %xmm10
12899        vaesenc	%xmm7, %xmm11, %xmm11
12900        vaesenc	%xmm7, %xmm12, %xmm12
12901        vaesenc	%xmm7, %xmm13, %xmm13
12902        vaesenc	%xmm7, %xmm14, %xmm14
12903        vaesenc	%xmm7, %xmm15, %xmm15
12904        vmovdqu	144(%rsi), %xmm7
12905        vaesenc	%xmm7, %xmm8, %xmm8
12906        vaesenc	%xmm7, %xmm9, %xmm9
12907        vaesenc	%xmm7, %xmm10, %xmm10
12908        vaesenc	%xmm7, %xmm11, %xmm11
12909        vaesenc	%xmm7, %xmm12, %xmm12
12910        vaesenc	%xmm7, %xmm13, %xmm13
12911        vaesenc	%xmm7, %xmm14, %xmm14
12912        vaesenc	%xmm7, %xmm15, %xmm15
12913        cmpl	$11, %r9d
12914        vmovdqu	160(%rsi), %xmm7
12915        jl	L_AES_GCM_encrypt_avx2_aesenc_128_enc_done
12916        vaesenc	%xmm7, %xmm8, %xmm8
12917        vaesenc	%xmm7, %xmm9, %xmm9
12918        vaesenc	%xmm7, %xmm10, %xmm10
12919        vaesenc	%xmm7, %xmm11, %xmm11
12920        vaesenc	%xmm7, %xmm12, %xmm12
12921        vaesenc	%xmm7, %xmm13, %xmm13
12922        vaesenc	%xmm7, %xmm14, %xmm14
12923        vaesenc	%xmm7, %xmm15, %xmm15
12924        vmovdqu	176(%rsi), %xmm7
12925        vaesenc	%xmm7, %xmm8, %xmm8
12926        vaesenc	%xmm7, %xmm9, %xmm9
12927        vaesenc	%xmm7, %xmm10, %xmm10
12928        vaesenc	%xmm7, %xmm11, %xmm11
12929        vaesenc	%xmm7, %xmm12, %xmm12
12930        vaesenc	%xmm7, %xmm13, %xmm13
12931        vaesenc	%xmm7, %xmm14, %xmm14
12932        vaesenc	%xmm7, %xmm15, %xmm15
12933        cmpl	$13, %r9d
12934        vmovdqu	192(%rsi), %xmm7
12935        jl	L_AES_GCM_encrypt_avx2_aesenc_128_enc_done
12936        vaesenc	%xmm7, %xmm8, %xmm8
12937        vaesenc	%xmm7, %xmm9, %xmm9
12938        vaesenc	%xmm7, %xmm10, %xmm10
12939        vaesenc	%xmm7, %xmm11, %xmm11
12940        vaesenc	%xmm7, %xmm12, %xmm12
12941        vaesenc	%xmm7, %xmm13, %xmm13
12942        vaesenc	%xmm7, %xmm14, %xmm14
12943        vaesenc	%xmm7, %xmm15, %xmm15
12944        vmovdqu	208(%rsi), %xmm7
12945        vaesenc	%xmm7, %xmm8, %xmm8
12946        vaesenc	%xmm7, %xmm9, %xmm9
12947        vaesenc	%xmm7, %xmm10, %xmm10
12948        vaesenc	%xmm7, %xmm11, %xmm11
12949        vaesenc	%xmm7, %xmm12, %xmm12
12950        vaesenc	%xmm7, %xmm13, %xmm13
12951        vaesenc	%xmm7, %xmm14, %xmm14
12952        vaesenc	%xmm7, %xmm15, %xmm15
12953        vmovdqu	224(%rsi), %xmm7
12954L_AES_GCM_encrypt_avx2_aesenc_128_enc_done:
12955        # aesenc_last
12956        vaesenclast	%xmm7, %xmm8, %xmm8
12957        vaesenclast	%xmm7, %xmm9, %xmm9
12958        vaesenclast	%xmm7, %xmm10, %xmm10
12959        vaesenclast	%xmm7, %xmm11, %xmm11
12960        vmovdqu	(%rdi), %xmm0
12961        vmovdqu	16(%rdi), %xmm1
12962        vmovdqu	32(%rdi), %xmm2
12963        vmovdqu	48(%rdi), %xmm3
12964        vpxor	%xmm0, %xmm8, %xmm8
12965        vpxor	%xmm1, %xmm9, %xmm9
12966        vpxor	%xmm2, %xmm10, %xmm10
12967        vpxor	%xmm3, %xmm11, %xmm11
12968        vmovdqu	%xmm8, (%r8)
12969        vmovdqu	%xmm9, 16(%r8)
12970        vmovdqu	%xmm10, 32(%r8)
12971        vmovdqu	%xmm11, 48(%r8)
12972        vaesenclast	%xmm7, %xmm12, %xmm12
12973        vaesenclast	%xmm7, %xmm13, %xmm13
12974        vaesenclast	%xmm7, %xmm14, %xmm14
12975        vaesenclast	%xmm7, %xmm15, %xmm15
12976        vmovdqu	64(%rdi), %xmm0
12977        vmovdqu	80(%rdi), %xmm1
12978        vmovdqu	96(%rdi), %xmm2
12979        vmovdqu	112(%rdi), %xmm3
12980        vpxor	%xmm0, %xmm12, %xmm12
12981        vpxor	%xmm1, %xmm13, %xmm13
12982        vpxor	%xmm2, %xmm14, %xmm14
12983        vpxor	%xmm3, %xmm15, %xmm15
12984        vmovdqu	%xmm12, 64(%r8)
12985        vmovdqu	%xmm13, 80(%r8)
12986        vmovdqu	%xmm14, 96(%r8)
12987        vmovdqu	%xmm15, 112(%r8)
12988        cmpl	$0x80, %r13d
12989        movl	$0x80, %ebx
12990        jle	L_AES_GCM_encrypt_avx2_end_128
12991        # More 128 bytes of input
12992L_AES_GCM_encrypt_avx2_ghash_128:
12993        # aesenc_128_ghash
12994        leaq	(%rdi,%rbx,1), %rcx
12995        leaq	(%r8,%rbx,1), %rdx
12996        # aesenc_ctr
12997        vmovdqu	128(%rsp), %xmm0
12998        vmovdqu	L_avx2_aes_gcm_bswap_epi64(%rip), %xmm1
12999        vpaddd	L_avx2_aes_gcm_one(%rip), %xmm0, %xmm9
13000        vpshufb	%xmm1, %xmm0, %xmm8
13001        vpaddd	L_avx2_aes_gcm_two(%rip), %xmm0, %xmm10
13002        vpshufb	%xmm1, %xmm9, %xmm9
13003        vpaddd	L_avx2_aes_gcm_three(%rip), %xmm0, %xmm11
13004        vpshufb	%xmm1, %xmm10, %xmm10
13005        vpaddd	L_avx2_aes_gcm_four(%rip), %xmm0, %xmm12
13006        vpshufb	%xmm1, %xmm11, %xmm11
13007        vpaddd	L_avx2_aes_gcm_five(%rip), %xmm0, %xmm13
13008        vpshufb	%xmm1, %xmm12, %xmm12
13009        vpaddd	L_avx2_aes_gcm_six(%rip), %xmm0, %xmm14
13010        vpshufb	%xmm1, %xmm13, %xmm13
13011        vpaddd	L_avx2_aes_gcm_seven(%rip), %xmm0, %xmm15
13012        vpshufb	%xmm1, %xmm14, %xmm14
13013        vpaddd	L_avx2_aes_gcm_eight(%rip), %xmm0, %xmm0
13014        vpshufb	%xmm1, %xmm15, %xmm15
13015        # aesenc_xor
13016        vmovdqu	(%rsi), %xmm7
13017        vmovdqu	%xmm0, 128(%rsp)
13018        vpxor	%xmm7, %xmm8, %xmm8
13019        vpxor	%xmm7, %xmm9, %xmm9
13020        vpxor	%xmm7, %xmm10, %xmm10
13021        vpxor	%xmm7, %xmm11, %xmm11
13022        vpxor	%xmm7, %xmm12, %xmm12
13023        vpxor	%xmm7, %xmm13, %xmm13
13024        vpxor	%xmm7, %xmm14, %xmm14
13025        vpxor	%xmm7, %xmm15, %xmm15
13026        # aesenc_pclmul_1
13027        vmovdqu	-128(%rdx), %xmm1
13028        vmovdqu	16(%rsi), %xmm0
13029        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm1, %xmm1
13030        vmovdqu	112(%rsp), %xmm2
13031        vpxor	%xmm6, %xmm1, %xmm1
13032        vpclmulqdq	$16, %xmm2, %xmm1, %xmm5
13033        vpclmulqdq	$0x01, %xmm2, %xmm1, %xmm3
13034        vpclmulqdq	$0x00, %xmm2, %xmm1, %xmm6
13035        vpclmulqdq	$0x11, %xmm2, %xmm1, %xmm7
13036        vaesenc	%xmm0, %xmm8, %xmm8
13037        vaesenc	%xmm0, %xmm9, %xmm9
13038        vaesenc	%xmm0, %xmm10, %xmm10
13039        vaesenc	%xmm0, %xmm11, %xmm11
13040        vaesenc	%xmm0, %xmm12, %xmm12
13041        vaesenc	%xmm0, %xmm13, %xmm13
13042        vaesenc	%xmm0, %xmm14, %xmm14
13043        vaesenc	%xmm0, %xmm15, %xmm15
13044        # aesenc_pclmul_2
13045        vmovdqu	-112(%rdx), %xmm1
13046        vmovdqu	96(%rsp), %xmm0
13047        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm1, %xmm1
13048        vpxor	%xmm3, %xmm5, %xmm5
13049        vpclmulqdq	$16, %xmm0, %xmm1, %xmm2
13050        vpclmulqdq	$0x01, %xmm0, %xmm1, %xmm3
13051        vpclmulqdq	$0x00, %xmm0, %xmm1, %xmm4
13052        vpclmulqdq	$0x11, %xmm0, %xmm1, %xmm1
13053        vmovdqu	32(%rsi), %xmm0
13054        vpxor	%xmm1, %xmm7, %xmm7
13055        vaesenc	%xmm0, %xmm8, %xmm8
13056        vaesenc	%xmm0, %xmm9, %xmm9
13057        vaesenc	%xmm0, %xmm10, %xmm10
13058        vaesenc	%xmm0, %xmm11, %xmm11
13059        vaesenc	%xmm0, %xmm12, %xmm12
13060        vaesenc	%xmm0, %xmm13, %xmm13
13061        vaesenc	%xmm0, %xmm14, %xmm14
13062        vaesenc	%xmm0, %xmm15, %xmm15
13063        # aesenc_pclmul_n
13064        vmovdqu	-96(%rdx), %xmm1
13065        vmovdqu	80(%rsp), %xmm0
13066        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm1, %xmm1
13067        vpxor	%xmm2, %xmm5, %xmm5
13068        vpclmulqdq	$16, %xmm0, %xmm1, %xmm2
13069        vpxor	%xmm3, %xmm5, %xmm5
13070        vpclmulqdq	$0x01, %xmm0, %xmm1, %xmm3
13071        vpxor	%xmm4, %xmm6, %xmm6
13072        vpclmulqdq	$0x00, %xmm0, %xmm1, %xmm4
13073        vpclmulqdq	$0x11, %xmm0, %xmm1, %xmm1
13074        vmovdqu	48(%rsi), %xmm0
13075        vpxor	%xmm1, %xmm7, %xmm7
13076        vaesenc	%xmm0, %xmm8, %xmm8
13077        vaesenc	%xmm0, %xmm9, %xmm9
13078        vaesenc	%xmm0, %xmm10, %xmm10
13079        vaesenc	%xmm0, %xmm11, %xmm11
13080        vaesenc	%xmm0, %xmm12, %xmm12
13081        vaesenc	%xmm0, %xmm13, %xmm13
13082        vaesenc	%xmm0, %xmm14, %xmm14
13083        vaesenc	%xmm0, %xmm15, %xmm15
13084        # aesenc_pclmul_n
13085        vmovdqu	-80(%rdx), %xmm1
13086        vmovdqu	64(%rsp), %xmm0
13087        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm1, %xmm1
13088        vpxor	%xmm2, %xmm5, %xmm5
13089        vpclmulqdq	$16, %xmm0, %xmm1, %xmm2
13090        vpxor	%xmm3, %xmm5, %xmm5
13091        vpclmulqdq	$0x01, %xmm0, %xmm1, %xmm3
13092        vpxor	%xmm4, %xmm6, %xmm6
13093        vpclmulqdq	$0x00, %xmm0, %xmm1, %xmm4
13094        vpclmulqdq	$0x11, %xmm0, %xmm1, %xmm1
13095        vmovdqu	64(%rsi), %xmm0
13096        vpxor	%xmm1, %xmm7, %xmm7
13097        vaesenc	%xmm0, %xmm8, %xmm8
13098        vaesenc	%xmm0, %xmm9, %xmm9
13099        vaesenc	%xmm0, %xmm10, %xmm10
13100        vaesenc	%xmm0, %xmm11, %xmm11
13101        vaesenc	%xmm0, %xmm12, %xmm12
13102        vaesenc	%xmm0, %xmm13, %xmm13
13103        vaesenc	%xmm0, %xmm14, %xmm14
13104        vaesenc	%xmm0, %xmm15, %xmm15
13105        # aesenc_pclmul_n
13106        vmovdqu	-64(%rdx), %xmm1
13107        vmovdqu	48(%rsp), %xmm0
13108        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm1, %xmm1
13109        vpxor	%xmm2, %xmm5, %xmm5
13110        vpclmulqdq	$16, %xmm0, %xmm1, %xmm2
13111        vpxor	%xmm3, %xmm5, %xmm5
13112        vpclmulqdq	$0x01, %xmm0, %xmm1, %xmm3
13113        vpxor	%xmm4, %xmm6, %xmm6
13114        vpclmulqdq	$0x00, %xmm0, %xmm1, %xmm4
13115        vpclmulqdq	$0x11, %xmm0, %xmm1, %xmm1
13116        vmovdqu	80(%rsi), %xmm0
13117        vpxor	%xmm1, %xmm7, %xmm7
13118        vaesenc	%xmm0, %xmm8, %xmm8
13119        vaesenc	%xmm0, %xmm9, %xmm9
13120        vaesenc	%xmm0, %xmm10, %xmm10
13121        vaesenc	%xmm0, %xmm11, %xmm11
13122        vaesenc	%xmm0, %xmm12, %xmm12
13123        vaesenc	%xmm0, %xmm13, %xmm13
13124        vaesenc	%xmm0, %xmm14, %xmm14
13125        vaesenc	%xmm0, %xmm15, %xmm15
13126        # aesenc_pclmul_n
13127        vmovdqu	-48(%rdx), %xmm1
13128        vmovdqu	32(%rsp), %xmm0
13129        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm1, %xmm1
13130        vpxor	%xmm2, %xmm5, %xmm5
13131        vpclmulqdq	$16, %xmm0, %xmm1, %xmm2
13132        vpxor	%xmm3, %xmm5, %xmm5
13133        vpclmulqdq	$0x01, %xmm0, %xmm1, %xmm3
13134        vpxor	%xmm4, %xmm6, %xmm6
13135        vpclmulqdq	$0x00, %xmm0, %xmm1, %xmm4
13136        vpclmulqdq	$0x11, %xmm0, %xmm1, %xmm1
13137        vmovdqu	96(%rsi), %xmm0
13138        vpxor	%xmm1, %xmm7, %xmm7
13139        vaesenc	%xmm0, %xmm8, %xmm8
13140        vaesenc	%xmm0, %xmm9, %xmm9
13141        vaesenc	%xmm0, %xmm10, %xmm10
13142        vaesenc	%xmm0, %xmm11, %xmm11
13143        vaesenc	%xmm0, %xmm12, %xmm12
13144        vaesenc	%xmm0, %xmm13, %xmm13
13145        vaesenc	%xmm0, %xmm14, %xmm14
13146        vaesenc	%xmm0, %xmm15, %xmm15
13147        # aesenc_pclmul_n
13148        vmovdqu	-32(%rdx), %xmm1
13149        vmovdqu	16(%rsp), %xmm0
13150        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm1, %xmm1
13151        vpxor	%xmm2, %xmm5, %xmm5
13152        vpclmulqdq	$16, %xmm0, %xmm1, %xmm2
13153        vpxor	%xmm3, %xmm5, %xmm5
13154        vpclmulqdq	$0x01, %xmm0, %xmm1, %xmm3
13155        vpxor	%xmm4, %xmm6, %xmm6
13156        vpclmulqdq	$0x00, %xmm0, %xmm1, %xmm4
13157        vpclmulqdq	$0x11, %xmm0, %xmm1, %xmm1
13158        vmovdqu	112(%rsi), %xmm0
13159        vpxor	%xmm1, %xmm7, %xmm7
13160        vaesenc	%xmm0, %xmm8, %xmm8
13161        vaesenc	%xmm0, %xmm9, %xmm9
13162        vaesenc	%xmm0, %xmm10, %xmm10
13163        vaesenc	%xmm0, %xmm11, %xmm11
13164        vaesenc	%xmm0, %xmm12, %xmm12
13165        vaesenc	%xmm0, %xmm13, %xmm13
13166        vaesenc	%xmm0, %xmm14, %xmm14
13167        vaesenc	%xmm0, %xmm15, %xmm15
13168        # aesenc_pclmul_n
13169        vmovdqu	-16(%rdx), %xmm1
13170        vmovdqu	(%rsp), %xmm0
13171        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm1, %xmm1
13172        vpxor	%xmm2, %xmm5, %xmm5
13173        vpclmulqdq	$16, %xmm0, %xmm1, %xmm2
13174        vpxor	%xmm3, %xmm5, %xmm5
13175        vpclmulqdq	$0x01, %xmm0, %xmm1, %xmm3
13176        vpxor	%xmm4, %xmm6, %xmm6
13177        vpclmulqdq	$0x00, %xmm0, %xmm1, %xmm4
13178        vpclmulqdq	$0x11, %xmm0, %xmm1, %xmm1
13179        vmovdqu	128(%rsi), %xmm0
13180        vpxor	%xmm1, %xmm7, %xmm7
13181        vaesenc	%xmm0, %xmm8, %xmm8
13182        vaesenc	%xmm0, %xmm9, %xmm9
13183        vaesenc	%xmm0, %xmm10, %xmm10
13184        vaesenc	%xmm0, %xmm11, %xmm11
13185        vaesenc	%xmm0, %xmm12, %xmm12
13186        vaesenc	%xmm0, %xmm13, %xmm13
13187        vaesenc	%xmm0, %xmm14, %xmm14
13188        vaesenc	%xmm0, %xmm15, %xmm15
13189        # aesenc_pclmul_l
13190        vpxor	%xmm2, %xmm5, %xmm5
13191        vpxor	%xmm4, %xmm6, %xmm6
13192        vpxor	%xmm3, %xmm5, %xmm5
13193        vpslldq	$8, %xmm5, %xmm1
13194        vpsrldq	$8, %xmm5, %xmm5
13195        vmovdqu	144(%rsi), %xmm4
13196        vmovdqu	L_avx2_aes_gcm_mod2_128(%rip), %xmm0
13197        vaesenc	%xmm4, %xmm8, %xmm8
13198        vpxor	%xmm1, %xmm6, %xmm6
13199        vpxor	%xmm5, %xmm7, %xmm7
13200        vpclmulqdq	$16, %xmm0, %xmm6, %xmm3
13201        vaesenc	%xmm4, %xmm9, %xmm9
13202        vaesenc	%xmm4, %xmm10, %xmm10
13203        vaesenc	%xmm4, %xmm11, %xmm11
13204        vpshufd	$0x4e, %xmm6, %xmm6
13205        vpxor	%xmm3, %xmm6, %xmm6
13206        vpclmulqdq	$16, %xmm0, %xmm6, %xmm3
13207        vaesenc	%xmm4, %xmm12, %xmm12
13208        vaesenc	%xmm4, %xmm13, %xmm13
13209        vaesenc	%xmm4, %xmm14, %xmm14
13210        vpshufd	$0x4e, %xmm6, %xmm6
13211        vpxor	%xmm3, %xmm6, %xmm6
13212        vpxor	%xmm7, %xmm6, %xmm6
13213        vaesenc	%xmm4, %xmm15, %xmm15
13214        cmpl	$11, %r9d
13215        vmovdqu	160(%rsi), %xmm7
13216        jl	L_AES_GCM_encrypt_avx2_aesenc_128_ghash_avx_done
13217        vaesenc	%xmm7, %xmm8, %xmm8
13218        vaesenc	%xmm7, %xmm9, %xmm9
13219        vaesenc	%xmm7, %xmm10, %xmm10
13220        vaesenc	%xmm7, %xmm11, %xmm11
13221        vaesenc	%xmm7, %xmm12, %xmm12
13222        vaesenc	%xmm7, %xmm13, %xmm13
13223        vaesenc	%xmm7, %xmm14, %xmm14
13224        vaesenc	%xmm7, %xmm15, %xmm15
13225        vmovdqu	176(%rsi), %xmm7
13226        vaesenc	%xmm7, %xmm8, %xmm8
13227        vaesenc	%xmm7, %xmm9, %xmm9
13228        vaesenc	%xmm7, %xmm10, %xmm10
13229        vaesenc	%xmm7, %xmm11, %xmm11
13230        vaesenc	%xmm7, %xmm12, %xmm12
13231        vaesenc	%xmm7, %xmm13, %xmm13
13232        vaesenc	%xmm7, %xmm14, %xmm14
13233        vaesenc	%xmm7, %xmm15, %xmm15
13234        cmpl	$13, %r9d
13235        vmovdqu	192(%rsi), %xmm7
13236        jl	L_AES_GCM_encrypt_avx2_aesenc_128_ghash_avx_done
13237        vaesenc	%xmm7, %xmm8, %xmm8
13238        vaesenc	%xmm7, %xmm9, %xmm9
13239        vaesenc	%xmm7, %xmm10, %xmm10
13240        vaesenc	%xmm7, %xmm11, %xmm11
13241        vaesenc	%xmm7, %xmm12, %xmm12
13242        vaesenc	%xmm7, %xmm13, %xmm13
13243        vaesenc	%xmm7, %xmm14, %xmm14
13244        vaesenc	%xmm7, %xmm15, %xmm15
13245        vmovdqu	208(%rsi), %xmm7
13246        vaesenc	%xmm7, %xmm8, %xmm8
13247        vaesenc	%xmm7, %xmm9, %xmm9
13248        vaesenc	%xmm7, %xmm10, %xmm10
13249        vaesenc	%xmm7, %xmm11, %xmm11
13250        vaesenc	%xmm7, %xmm12, %xmm12
13251        vaesenc	%xmm7, %xmm13, %xmm13
13252        vaesenc	%xmm7, %xmm14, %xmm14
13253        vaesenc	%xmm7, %xmm15, %xmm15
13254        vmovdqu	224(%rsi), %xmm7
13255L_AES_GCM_encrypt_avx2_aesenc_128_ghash_avx_done:
13256        # aesenc_last
13257        vaesenclast	%xmm7, %xmm8, %xmm8
13258        vaesenclast	%xmm7, %xmm9, %xmm9
13259        vaesenclast	%xmm7, %xmm10, %xmm10
13260        vaesenclast	%xmm7, %xmm11, %xmm11
13261        vmovdqu	(%rcx), %xmm0
13262        vmovdqu	16(%rcx), %xmm1
13263        vmovdqu	32(%rcx), %xmm2
13264        vmovdqu	48(%rcx), %xmm3
13265        vpxor	%xmm0, %xmm8, %xmm8
13266        vpxor	%xmm1, %xmm9, %xmm9
13267        vpxor	%xmm2, %xmm10, %xmm10
13268        vpxor	%xmm3, %xmm11, %xmm11
13269        vmovdqu	%xmm8, (%rdx)
13270        vmovdqu	%xmm9, 16(%rdx)
13271        vmovdqu	%xmm10, 32(%rdx)
13272        vmovdqu	%xmm11, 48(%rdx)
13273        vaesenclast	%xmm7, %xmm12, %xmm12
13274        vaesenclast	%xmm7, %xmm13, %xmm13
13275        vaesenclast	%xmm7, %xmm14, %xmm14
13276        vaesenclast	%xmm7, %xmm15, %xmm15
13277        vmovdqu	64(%rcx), %xmm0
13278        vmovdqu	80(%rcx), %xmm1
13279        vmovdqu	96(%rcx), %xmm2
13280        vmovdqu	112(%rcx), %xmm3
13281        vpxor	%xmm0, %xmm12, %xmm12
13282        vpxor	%xmm1, %xmm13, %xmm13
13283        vpxor	%xmm2, %xmm14, %xmm14
13284        vpxor	%xmm3, %xmm15, %xmm15
13285        vmovdqu	%xmm12, 64(%rdx)
13286        vmovdqu	%xmm13, 80(%rdx)
13287        vmovdqu	%xmm14, 96(%rdx)
13288        vmovdqu	%xmm15, 112(%rdx)
13289        # aesenc_128_ghash - end
13290        addl	$0x80, %ebx
13291        cmpl	%r13d, %ebx
13292        jl	L_AES_GCM_encrypt_avx2_ghash_128
13293L_AES_GCM_encrypt_avx2_end_128:
13294        vmovdqu	L_avx2_aes_gcm_bswap_mask(%rip), %xmm4
13295        vpshufb	%xmm4, %xmm8, %xmm8
13296        vpshufb	%xmm4, %xmm9, %xmm9
13297        vpshufb	%xmm4, %xmm10, %xmm10
13298        vpshufb	%xmm4, %xmm11, %xmm11
13299        vpshufb	%xmm4, %xmm12, %xmm12
13300        vpshufb	%xmm4, %xmm13, %xmm13
13301        vpshufb	%xmm4, %xmm14, %xmm14
13302        vpshufb	%xmm4, %xmm15, %xmm15
13303        vpxor	%xmm6, %xmm8, %xmm8
13304        vmovdqu	(%rsp), %xmm7
13305        vpclmulqdq	$16, %xmm15, %xmm7, %xmm5
13306        vpclmulqdq	$0x01, %xmm15, %xmm7, %xmm1
13307        vpclmulqdq	$0x00, %xmm15, %xmm7, %xmm4
13308        vpclmulqdq	$0x11, %xmm15, %xmm7, %xmm6
13309        vpxor	%xmm1, %xmm5, %xmm5
13310        vmovdqu	16(%rsp), %xmm7
13311        vpclmulqdq	$16, %xmm14, %xmm7, %xmm2
13312        vpclmulqdq	$0x01, %xmm14, %xmm7, %xmm1
13313        vpclmulqdq	$0x00, %xmm14, %xmm7, %xmm0
13314        vpclmulqdq	$0x11, %xmm14, %xmm7, %xmm3
13315        vpxor	%xmm1, %xmm2, %xmm2
13316        vpxor	%xmm3, %xmm6, %xmm6
13317        vpxor	%xmm2, %xmm5, %xmm5
13318        vpxor	%xmm0, %xmm4, %xmm4
13319        vmovdqu	32(%rsp), %xmm15
13320        vmovdqu	48(%rsp), %xmm7
13321        vpclmulqdq	$16, %xmm13, %xmm15, %xmm2
13322        vpclmulqdq	$0x01, %xmm13, %xmm15, %xmm1
13323        vpclmulqdq	$0x00, %xmm13, %xmm15, %xmm0
13324        vpclmulqdq	$0x11, %xmm13, %xmm15, %xmm3
13325        vpxor	%xmm1, %xmm2, %xmm2
13326        vpxor	%xmm3, %xmm6, %xmm6
13327        vpxor	%xmm2, %xmm5, %xmm5
13328        vpxor	%xmm0, %xmm4, %xmm4
13329        vpclmulqdq	$16, %xmm12, %xmm7, %xmm2
13330        vpclmulqdq	$0x01, %xmm12, %xmm7, %xmm1
13331        vpclmulqdq	$0x00, %xmm12, %xmm7, %xmm0
13332        vpclmulqdq	$0x11, %xmm12, %xmm7, %xmm3
13333        vpxor	%xmm1, %xmm2, %xmm2
13334        vpxor	%xmm3, %xmm6, %xmm6
13335        vpxor	%xmm2, %xmm5, %xmm5
13336        vpxor	%xmm0, %xmm4, %xmm4
13337        vmovdqu	64(%rsp), %xmm15
13338        vmovdqu	80(%rsp), %xmm7
13339        vpclmulqdq	$16, %xmm11, %xmm15, %xmm2
13340        vpclmulqdq	$0x01, %xmm11, %xmm15, %xmm1
13341        vpclmulqdq	$0x00, %xmm11, %xmm15, %xmm0
13342        vpclmulqdq	$0x11, %xmm11, %xmm15, %xmm3
13343        vpxor	%xmm1, %xmm2, %xmm2
13344        vpxor	%xmm3, %xmm6, %xmm6
13345        vpxor	%xmm2, %xmm5, %xmm5
13346        vpxor	%xmm0, %xmm4, %xmm4
13347        vpclmulqdq	$16, %xmm10, %xmm7, %xmm2
13348        vpclmulqdq	$0x01, %xmm10, %xmm7, %xmm1
13349        vpclmulqdq	$0x00, %xmm10, %xmm7, %xmm0
13350        vpclmulqdq	$0x11, %xmm10, %xmm7, %xmm3
13351        vpxor	%xmm1, %xmm2, %xmm2
13352        vpxor	%xmm3, %xmm6, %xmm6
13353        vpxor	%xmm2, %xmm5, %xmm5
13354        vpxor	%xmm0, %xmm4, %xmm4
13355        vmovdqu	96(%rsp), %xmm15
13356        vmovdqu	112(%rsp), %xmm7
13357        vpclmulqdq	$16, %xmm9, %xmm15, %xmm2
13358        vpclmulqdq	$0x01, %xmm9, %xmm15, %xmm1
13359        vpclmulqdq	$0x00, %xmm9, %xmm15, %xmm0
13360        vpclmulqdq	$0x11, %xmm9, %xmm15, %xmm3
13361        vpxor	%xmm1, %xmm2, %xmm2
13362        vpxor	%xmm3, %xmm6, %xmm6
13363        vpxor	%xmm2, %xmm5, %xmm5
13364        vpxor	%xmm0, %xmm4, %xmm4
13365        vpclmulqdq	$16, %xmm8, %xmm7, %xmm2
13366        vpclmulqdq	$0x01, %xmm8, %xmm7, %xmm1
13367        vpclmulqdq	$0x00, %xmm8, %xmm7, %xmm0
13368        vpclmulqdq	$0x11, %xmm8, %xmm7, %xmm3
13369        vpxor	%xmm1, %xmm2, %xmm2
13370        vpxor	%xmm3, %xmm6, %xmm6
13371        vpxor	%xmm2, %xmm5, %xmm5
13372        vpxor	%xmm0, %xmm4, %xmm4
13373        vpslldq	$8, %xmm5, %xmm7
13374        vpsrldq	$8, %xmm5, %xmm5
13375        vpxor	%xmm7, %xmm4, %xmm4
13376        vpxor	%xmm5, %xmm6, %xmm6
13377        # ghash_red
13378        vmovdqu	L_avx2_aes_gcm_mod2_128(%rip), %xmm2
13379        vpclmulqdq	$16, %xmm2, %xmm4, %xmm0
13380        vpshufd	$0x4e, %xmm4, %xmm1
13381        vpxor	%xmm0, %xmm1, %xmm1
13382        vpclmulqdq	$16, %xmm2, %xmm1, %xmm0
13383        vpshufd	$0x4e, %xmm1, %xmm1
13384        vpxor	%xmm0, %xmm1, %xmm1
13385        vpxor	%xmm1, %xmm6, %xmm6
13386        vmovdqu	(%rsp), %xmm5
13387        vmovdqu	128(%rsp), %xmm4
13388        vmovdqu	144(%rsp), %xmm15
13389L_AES_GCM_encrypt_avx2_done_128:
13390        cmpl	%r10d, %ebx
13391        je	L_AES_GCM_encrypt_avx2_done_enc
13392        movl	%r10d, %r13d
13393        andl	$0xfffffff0, %r13d
13394        cmpl	%r13d, %ebx
13395        jge	L_AES_GCM_encrypt_avx2_last_block_done
13396        # aesenc_block
13397        vmovdqu	%xmm4, %xmm1
13398        vpshufb	L_avx2_aes_gcm_bswap_epi64(%rip), %xmm1, %xmm0
13399        vpaddd	L_avx2_aes_gcm_one(%rip), %xmm1, %xmm1
13400        vpxor	(%rsi), %xmm0, %xmm0
13401        vmovdqu	16(%rsi), %xmm2
13402        vaesenc	%xmm2, %xmm0, %xmm0
13403        vmovdqu	32(%rsi), %xmm2
13404        vaesenc	%xmm2, %xmm0, %xmm0
13405        vmovdqu	48(%rsi), %xmm2
13406        vaesenc	%xmm2, %xmm0, %xmm0
13407        vmovdqu	64(%rsi), %xmm2
13408        vaesenc	%xmm2, %xmm0, %xmm0
13409        vmovdqu	80(%rsi), %xmm2
13410        vaesenc	%xmm2, %xmm0, %xmm0
13411        vmovdqu	96(%rsi), %xmm2
13412        vaesenc	%xmm2, %xmm0, %xmm0
13413        vmovdqu	112(%rsi), %xmm2
13414        vaesenc	%xmm2, %xmm0, %xmm0
13415        vmovdqu	128(%rsi), %xmm2
13416        vaesenc	%xmm2, %xmm0, %xmm0
13417        vmovdqu	144(%rsi), %xmm2
13418        vaesenc	%xmm2, %xmm0, %xmm0
13419        vmovdqu	%xmm1, %xmm4
13420        cmpl	$11, %r9d
13421        vmovdqu	160(%rsi), %xmm1
13422        jl	L_AES_GCM_encrypt_avx2_aesenc_block_last
13423        vaesenc	%xmm1, %xmm0, %xmm0
13424        vmovdqu	176(%rsi), %xmm2
13425        vaesenc	%xmm2, %xmm0, %xmm0
13426        cmpl	$13, %r9d
13427        vmovdqu	192(%rsi), %xmm1
13428        jl	L_AES_GCM_encrypt_avx2_aesenc_block_last
13429        vaesenc	%xmm1, %xmm0, %xmm0
13430        vmovdqu	208(%rsi), %xmm2
13431        vaesenc	%xmm2, %xmm0, %xmm0
13432        vmovdqu	224(%rsi), %xmm1
13433L_AES_GCM_encrypt_avx2_aesenc_block_last:
13434        vaesenclast	%xmm1, %xmm0, %xmm0
13435        vmovdqu	(%rdi,%rbx,1), %xmm1
13436        vpxor	%xmm1, %xmm0, %xmm0
13437        vmovdqu	%xmm0, (%r8,%rbx,1)
13438        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0
13439        vpxor	%xmm0, %xmm6, %xmm6
13440        addl	$16, %ebx
13441        cmpl	%r13d, %ebx
13442        jge	L_AES_GCM_encrypt_avx2_last_block_ghash
13443L_AES_GCM_encrypt_avx2_last_block_start:
13444        vmovdqu	(%rdi,%rbx,1), %xmm12
13445        vpshufb	L_avx2_aes_gcm_bswap_epi64(%rip), %xmm4, %xmm11
13446        vpaddd	L_avx2_aes_gcm_one(%rip), %xmm4, %xmm4
13447        # aesenc_gfmul_sb
13448        vpclmulqdq	$0x01, %xmm5, %xmm6, %xmm2
13449        vpclmulqdq	$16, %xmm5, %xmm6, %xmm3
13450        vpclmulqdq	$0x00, %xmm5, %xmm6, %xmm1
13451        vpclmulqdq	$0x11, %xmm5, %xmm6, %xmm8
13452        vpxor	(%rsi), %xmm11, %xmm11
13453        vaesenc	16(%rsi), %xmm11, %xmm11
13454        vpxor	%xmm2, %xmm3, %xmm3
13455        vpslldq	$8, %xmm3, %xmm2
13456        vpsrldq	$8, %xmm3, %xmm3
13457        vaesenc	32(%rsi), %xmm11, %xmm11
13458        vpxor	%xmm1, %xmm2, %xmm2
13459        vpclmulqdq	$16, L_avx2_aes_gcm_mod2_128(%rip), %xmm2, %xmm1
13460        vaesenc	48(%rsi), %xmm11, %xmm11
13461        vaesenc	64(%rsi), %xmm11, %xmm11
13462        vaesenc	80(%rsi), %xmm11, %xmm11
13463        vpshufd	$0x4e, %xmm2, %xmm2
13464        vpxor	%xmm1, %xmm2, %xmm2
13465        vpclmulqdq	$16, L_avx2_aes_gcm_mod2_128(%rip), %xmm2, %xmm1
13466        vaesenc	96(%rsi), %xmm11, %xmm11
13467        vaesenc	112(%rsi), %xmm11, %xmm11
13468        vaesenc	128(%rsi), %xmm11, %xmm11
13469        vpshufd	$0x4e, %xmm2, %xmm2
13470        vaesenc	144(%rsi), %xmm11, %xmm11
13471        vpxor	%xmm3, %xmm8, %xmm8
13472        vpxor	%xmm8, %xmm2, %xmm2
13473        vmovdqu	160(%rsi), %xmm0
13474        cmpl	$11, %r9d
13475        jl	L_AES_GCM_encrypt_avx2_aesenc_gfmul_sb_last
13476        vaesenc	%xmm0, %xmm11, %xmm11
13477        vaesenc	176(%rsi), %xmm11, %xmm11
13478        vmovdqu	192(%rsi), %xmm0
13479        cmpl	$13, %r9d
13480        jl	L_AES_GCM_encrypt_avx2_aesenc_gfmul_sb_last
13481        vaesenc	%xmm0, %xmm11, %xmm11
13482        vaesenc	208(%rsi), %xmm11, %xmm11
13483        vmovdqu	224(%rsi), %xmm0
13484L_AES_GCM_encrypt_avx2_aesenc_gfmul_sb_last:
13485        vaesenclast	%xmm0, %xmm11, %xmm11
13486        vpxor	%xmm1, %xmm2, %xmm6
13487        vpxor	%xmm12, %xmm11, %xmm11
13488        vmovdqu	%xmm11, (%r8,%rbx,1)
13489        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm11, %xmm11
13490        vpxor	%xmm11, %xmm6, %xmm6
13491        addl	$16, %ebx
13492        cmpl	%r13d, %ebx
13493        jl	L_AES_GCM_encrypt_avx2_last_block_start
13494L_AES_GCM_encrypt_avx2_last_block_ghash:
13495        # ghash_gfmul_red
13496        vpclmulqdq	$16, %xmm5, %xmm6, %xmm10
13497        vpclmulqdq	$0x01, %xmm5, %xmm6, %xmm9
13498        vpclmulqdq	$0x00, %xmm5, %xmm6, %xmm8
13499        vpxor	%xmm9, %xmm10, %xmm10
13500        vpslldq	$8, %xmm10, %xmm9
13501        vpsrldq	$8, %xmm10, %xmm10
13502        vpxor	%xmm8, %xmm9, %xmm9
13503        vpclmulqdq	$0x11, %xmm5, %xmm6, %xmm6
13504        vpclmulqdq	$16, L_avx2_aes_gcm_mod2_128(%rip), %xmm9, %xmm8
13505        vpshufd	$0x4e, %xmm9, %xmm9
13506        vpxor	%xmm8, %xmm9, %xmm9
13507        vpclmulqdq	$16, L_avx2_aes_gcm_mod2_128(%rip), %xmm9, %xmm8
13508        vpshufd	$0x4e, %xmm9, %xmm9
13509        vpxor	%xmm10, %xmm6, %xmm6
13510        vpxor	%xmm9, %xmm6, %xmm6
13511        vpxor	%xmm8, %xmm6, %xmm6
13512L_AES_GCM_encrypt_avx2_last_block_done:
13513        movl	%r10d, %ecx
13514        movl	%r10d, %edx
13515        andl	$15, %ecx
13516        jz	L_AES_GCM_encrypt_avx2_done_enc
13517        # aesenc_last15_enc
13518        vpshufb	L_avx2_aes_gcm_bswap_epi64(%rip), %xmm4, %xmm4
13519        vpxor	(%rsi), %xmm4, %xmm4
13520        vaesenc	16(%rsi), %xmm4, %xmm4
13521        vaesenc	32(%rsi), %xmm4, %xmm4
13522        vaesenc	48(%rsi), %xmm4, %xmm4
13523        vaesenc	64(%rsi), %xmm4, %xmm4
13524        vaesenc	80(%rsi), %xmm4, %xmm4
13525        vaesenc	96(%rsi), %xmm4, %xmm4
13526        vaesenc	112(%rsi), %xmm4, %xmm4
13527        vaesenc	128(%rsi), %xmm4, %xmm4
13528        vaesenc	144(%rsi), %xmm4, %xmm4
13529        cmpl	$11, %r9d
13530        vmovdqu	160(%rsi), %xmm0
13531        jl	L_AES_GCM_encrypt_avx2_aesenc_last15_enc_avx_aesenc_avx_last
13532        vaesenc	%xmm0, %xmm4, %xmm4
13533        vaesenc	176(%rsi), %xmm4, %xmm4
13534        cmpl	$13, %r9d
13535        vmovdqu	192(%rsi), %xmm0
13536        jl	L_AES_GCM_encrypt_avx2_aesenc_last15_enc_avx_aesenc_avx_last
13537        vaesenc	%xmm0, %xmm4, %xmm4
13538        vaesenc	208(%rsi), %xmm4, %xmm4
13539        vmovdqu	224(%rsi), %xmm0
13540L_AES_GCM_encrypt_avx2_aesenc_last15_enc_avx_aesenc_avx_last:
13541        vaesenclast	%xmm0, %xmm4, %xmm4
13542        xorl	%ecx, %ecx
13543        vpxor	%xmm0, %xmm0, %xmm0
13544        vmovdqu	%xmm4, (%rsp)
13545        vmovdqu	%xmm0, 16(%rsp)
13546L_AES_GCM_encrypt_avx2_aesenc_last15_enc_avx_loop:
13547        movzbl	(%rdi,%rbx,1), %r13d
13548        xorb	(%rsp,%rcx,1), %r13b
13549        movb	%r13b, 16(%rsp,%rcx,1)
13550        movb	%r13b, (%r8,%rbx,1)
13551        incl	%ebx
13552        incl	%ecx
13553        cmpl	%edx, %ebx
13554        jl	L_AES_GCM_encrypt_avx2_aesenc_last15_enc_avx_loop
13555L_AES_GCM_encrypt_avx2_aesenc_last15_enc_avx_finish_enc:
13556        vmovdqu	16(%rsp), %xmm4
13557        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm4, %xmm4
13558        vpxor	%xmm4, %xmm6, %xmm6
13559        # ghash_gfmul_red
13560        vpclmulqdq	$16, %xmm5, %xmm6, %xmm2
13561        vpclmulqdq	$0x01, %xmm5, %xmm6, %xmm1
13562        vpclmulqdq	$0x00, %xmm5, %xmm6, %xmm0
13563        vpxor	%xmm1, %xmm2, %xmm2
13564        vpslldq	$8, %xmm2, %xmm1
13565        vpsrldq	$8, %xmm2, %xmm2
13566        vpxor	%xmm0, %xmm1, %xmm1
13567        vpclmulqdq	$0x11, %xmm5, %xmm6, %xmm6
13568        vpclmulqdq	$16, L_avx2_aes_gcm_mod2_128(%rip), %xmm1, %xmm0
13569        vpshufd	$0x4e, %xmm1, %xmm1
13570        vpxor	%xmm0, %xmm1, %xmm1
13571        vpclmulqdq	$16, L_avx2_aes_gcm_mod2_128(%rip), %xmm1, %xmm0
13572        vpshufd	$0x4e, %xmm1, %xmm1
13573        vpxor	%xmm2, %xmm6, %xmm6
13574        vpxor	%xmm1, %xmm6, %xmm6
13575        vpxor	%xmm0, %xmm6, %xmm6
13576L_AES_GCM_encrypt_avx2_done_enc:
13577        # calc_tag
13578        shlq	$3, %r10
13579        shlq	$3, %r11
13580        vmovq	%r10, %xmm0
13581        vmovq	%r11, %xmm1
13582        vpunpcklqdq	%xmm1, %xmm0, %xmm0
13583        vpxor	%xmm6, %xmm0, %xmm0
13584        # ghash_gfmul_red
13585        vpclmulqdq	$16, %xmm5, %xmm0, %xmm4
13586        vpclmulqdq	$0x01, %xmm5, %xmm0, %xmm3
13587        vpclmulqdq	$0x00, %xmm5, %xmm0, %xmm2
13588        vpxor	%xmm3, %xmm4, %xmm4
13589        vpslldq	$8, %xmm4, %xmm3
13590        vpsrldq	$8, %xmm4, %xmm4
13591        vpxor	%xmm2, %xmm3, %xmm3
13592        vpclmulqdq	$0x11, %xmm5, %xmm0, %xmm0
13593        vpclmulqdq	$16, L_avx2_aes_gcm_mod2_128(%rip), %xmm3, %xmm2
13594        vpshufd	$0x4e, %xmm3, %xmm3
13595        vpxor	%xmm2, %xmm3, %xmm3
13596        vpclmulqdq	$16, L_avx2_aes_gcm_mod2_128(%rip), %xmm3, %xmm2
13597        vpshufd	$0x4e, %xmm3, %xmm3
13598        vpxor	%xmm4, %xmm0, %xmm0
13599        vpxor	%xmm3, %xmm0, %xmm0
13600        vpxor	%xmm2, %xmm0, %xmm0
13601        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0
13602        vpxor	%xmm15, %xmm0, %xmm0
13603        # store_tag
13604        cmpl	$16, %r14d
13605        je	L_AES_GCM_encrypt_avx2_store_tag_16
13606        xorq	%rcx, %rcx
13607        vmovdqu	%xmm0, (%rsp)
13608L_AES_GCM_encrypt_avx2_store_tag_loop:
13609        movzbl	(%rsp,%rcx,1), %r13d
13610        movb	%r13b, (%r15,%rcx,1)
13611        incl	%ecx
13612        cmpl	%r14d, %ecx
13613        jne	L_AES_GCM_encrypt_avx2_store_tag_loop
13614        jmp	L_AES_GCM_encrypt_avx2_store_tag_done
13615L_AES_GCM_encrypt_avx2_store_tag_16:
13616        vmovdqu	%xmm0, (%r15)
13617L_AES_GCM_encrypt_avx2_store_tag_done:
13618        vzeroupper
13619        addq	$0xa0, %rsp
13620        popq	%r14
13621        popq	%rbx
13622        popq	%r15
13623        popq	%r12
13624        popq	%r13
13625        repz retq
13626#ifndef __APPLE__
13627.size	AES_GCM_encrypt_avx2,.-AES_GCM_encrypt_avx2
13628#endif /* __APPLE__ */
13629#ifndef __APPLE__
13630.text
13631.globl	AES_GCM_decrypt_avx2
13632.type	AES_GCM_decrypt_avx2,@function
13633.align	16
13634AES_GCM_decrypt_avx2:
13635#else
13636.section	__TEXT,__text
13637.globl	_AES_GCM_decrypt_avx2
13638.p2align	4
13639_AES_GCM_decrypt_avx2:
13640#endif /* __APPLE__ */
13641        pushq	%r13
13642        pushq	%r12
13643        pushq	%r14
13644        pushq	%rbx
13645        pushq	%r15
13646        pushq	%rbp
13647        movq	%rdx, %r12
13648        movq	%rcx, %rax
13649        movq	%r8, %r14
13650        movq	%rsi, %r8
13651        movl	%r9d, %r10d
13652        movl	56(%rsp), %r11d
13653        movl	64(%rsp), %ebx
13654        movl	72(%rsp), %r15d
13655        movq	80(%rsp), %rsi
13656        movl	88(%rsp), %r9d
13657        movq	96(%rsp), %rbp
13658        subq	$0xa8, %rsp
13659        vpxor	%xmm4, %xmm4, %xmm4
13660        vpxor	%xmm6, %xmm6, %xmm6
13661        movl	%ebx, %edx
13662        cmpl	$12, %edx
13663        je	L_AES_GCM_decrypt_avx2_iv_12
13664        # Calculate values when IV is not 12 bytes
13665        # H = Encrypt X(=0)
13666        vmovdqu	(%rsi), %xmm5
13667        vaesenc	16(%rsi), %xmm5, %xmm5
13668        vaesenc	32(%rsi), %xmm5, %xmm5
13669        vaesenc	48(%rsi), %xmm5, %xmm5
13670        vaesenc	64(%rsi), %xmm5, %xmm5
13671        vaesenc	80(%rsi), %xmm5, %xmm5
13672        vaesenc	96(%rsi), %xmm5, %xmm5
13673        vaesenc	112(%rsi), %xmm5, %xmm5
13674        vaesenc	128(%rsi), %xmm5, %xmm5
13675        vaesenc	144(%rsi), %xmm5, %xmm5
13676        cmpl	$11, %r9d
13677        vmovdqu	160(%rsi), %xmm0
13678        jl	L_AES_GCM_decrypt_avx2_calc_iv_1_aesenc_avx_last
13679        vaesenc	%xmm0, %xmm5, %xmm5
13680        vaesenc	176(%rsi), %xmm5, %xmm5
13681        cmpl	$13, %r9d
13682        vmovdqu	192(%rsi), %xmm0
13683        jl	L_AES_GCM_decrypt_avx2_calc_iv_1_aesenc_avx_last
13684        vaesenc	%xmm0, %xmm5, %xmm5
13685        vaesenc	208(%rsi), %xmm5, %xmm5
13686        vmovdqu	224(%rsi), %xmm0
13687L_AES_GCM_decrypt_avx2_calc_iv_1_aesenc_avx_last:
13688        vaesenclast	%xmm0, %xmm5, %xmm5
13689        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm5, %xmm5
13690        # Calc counter
13691        # Initialization vector
13692        cmpl	$0x00, %edx
13693        movq	$0x00, %rcx
13694        je	L_AES_GCM_decrypt_avx2_calc_iv_done
13695        cmpl	$16, %edx
13696        jl	L_AES_GCM_decrypt_avx2_calc_iv_lt16
13697        andl	$0xfffffff0, %edx
13698L_AES_GCM_decrypt_avx2_calc_iv_16_loop:
13699        vmovdqu	(%rax,%rcx,1), %xmm0
13700        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0
13701        vpxor	%xmm0, %xmm4, %xmm4
13702        # ghash_gfmul_avx
13703        vpclmulqdq	$16, %xmm4, %xmm5, %xmm2
13704        vpclmulqdq	$0x01, %xmm4, %xmm5, %xmm1
13705        vpclmulqdq	$0x00, %xmm4, %xmm5, %xmm0
13706        vpclmulqdq	$0x11, %xmm4, %xmm5, %xmm3
13707        vpxor	%xmm1, %xmm2, %xmm2
13708        vpslldq	$8, %xmm2, %xmm1
13709        vpsrldq	$8, %xmm2, %xmm2
13710        vpxor	%xmm1, %xmm0, %xmm7
13711        vpxor	%xmm2, %xmm3, %xmm4
13712        # ghash_mid
13713        vpsrld	$31, %xmm7, %xmm0
13714        vpsrld	$31, %xmm4, %xmm1
13715        vpslld	$0x01, %xmm7, %xmm7
13716        vpslld	$0x01, %xmm4, %xmm4
13717        vpsrldq	$12, %xmm0, %xmm2
13718        vpslldq	$4, %xmm0, %xmm0
13719        vpslldq	$4, %xmm1, %xmm1
13720        vpor	%xmm2, %xmm4, %xmm4
13721        vpor	%xmm0, %xmm7, %xmm7
13722        vpor	%xmm1, %xmm4, %xmm4
13723        # ghash_red
13724        vmovdqu	L_avx2_aes_gcm_mod2_128(%rip), %xmm2
13725        vpclmulqdq	$16, %xmm2, %xmm7, %xmm0
13726        vpshufd	$0x4e, %xmm7, %xmm1
13727        vpxor	%xmm0, %xmm1, %xmm1
13728        vpclmulqdq	$16, %xmm2, %xmm1, %xmm0
13729        vpshufd	$0x4e, %xmm1, %xmm1
13730        vpxor	%xmm0, %xmm1, %xmm1
13731        vpxor	%xmm1, %xmm4, %xmm4
13732        addl	$16, %ecx
13733        cmpl	%edx, %ecx
13734        jl	L_AES_GCM_decrypt_avx2_calc_iv_16_loop
13735        movl	%ebx, %edx
13736        cmpl	%edx, %ecx
13737        je	L_AES_GCM_decrypt_avx2_calc_iv_done
13738L_AES_GCM_decrypt_avx2_calc_iv_lt16:
13739        vpxor	%xmm0, %xmm0, %xmm0
13740        xorl	%ebx, %ebx
13741        vmovdqu	%xmm0, (%rsp)
13742L_AES_GCM_decrypt_avx2_calc_iv_loop:
13743        movzbl	(%rax,%rcx,1), %r13d
13744        movb	%r13b, (%rsp,%rbx,1)
13745        incl	%ecx
13746        incl	%ebx
13747        cmpl	%edx, %ecx
13748        jl	L_AES_GCM_decrypt_avx2_calc_iv_loop
13749        vmovdqu	(%rsp), %xmm0
13750        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0
13751        vpxor	%xmm0, %xmm4, %xmm4
13752        # ghash_gfmul_avx
13753        vpclmulqdq	$16, %xmm4, %xmm5, %xmm2
13754        vpclmulqdq	$0x01, %xmm4, %xmm5, %xmm1
13755        vpclmulqdq	$0x00, %xmm4, %xmm5, %xmm0
13756        vpclmulqdq	$0x11, %xmm4, %xmm5, %xmm3
13757        vpxor	%xmm1, %xmm2, %xmm2
13758        vpslldq	$8, %xmm2, %xmm1
13759        vpsrldq	$8, %xmm2, %xmm2
13760        vpxor	%xmm1, %xmm0, %xmm7
13761        vpxor	%xmm2, %xmm3, %xmm4
13762        # ghash_mid
13763        vpsrld	$31, %xmm7, %xmm0
13764        vpsrld	$31, %xmm4, %xmm1
13765        vpslld	$0x01, %xmm7, %xmm7
13766        vpslld	$0x01, %xmm4, %xmm4
13767        vpsrldq	$12, %xmm0, %xmm2
13768        vpslldq	$4, %xmm0, %xmm0
13769        vpslldq	$4, %xmm1, %xmm1
13770        vpor	%xmm2, %xmm4, %xmm4
13771        vpor	%xmm0, %xmm7, %xmm7
13772        vpor	%xmm1, %xmm4, %xmm4
13773        # ghash_red
13774        vmovdqu	L_avx2_aes_gcm_mod2_128(%rip), %xmm2
13775        vpclmulqdq	$16, %xmm2, %xmm7, %xmm0
13776        vpshufd	$0x4e, %xmm7, %xmm1
13777        vpxor	%xmm0, %xmm1, %xmm1
13778        vpclmulqdq	$16, %xmm2, %xmm1, %xmm0
13779        vpshufd	$0x4e, %xmm1, %xmm1
13780        vpxor	%xmm0, %xmm1, %xmm1
13781        vpxor	%xmm1, %xmm4, %xmm4
13782L_AES_GCM_decrypt_avx2_calc_iv_done:
13783        # T = Encrypt counter
13784        vpxor	%xmm0, %xmm0, %xmm0
13785        shll	$3, %edx
13786        vmovq	%rdx, %xmm0
13787        vpxor	%xmm0, %xmm4, %xmm4
13788        # ghash_gfmul_avx
13789        vpclmulqdq	$16, %xmm4, %xmm5, %xmm2
13790        vpclmulqdq	$0x01, %xmm4, %xmm5, %xmm1
13791        vpclmulqdq	$0x00, %xmm4, %xmm5, %xmm0
13792        vpclmulqdq	$0x11, %xmm4, %xmm5, %xmm3
13793        vpxor	%xmm1, %xmm2, %xmm2
13794        vpslldq	$8, %xmm2, %xmm1
13795        vpsrldq	$8, %xmm2, %xmm2
13796        vpxor	%xmm1, %xmm0, %xmm7
13797        vpxor	%xmm2, %xmm3, %xmm4
13798        # ghash_mid
13799        vpsrld	$31, %xmm7, %xmm0
13800        vpsrld	$31, %xmm4, %xmm1
13801        vpslld	$0x01, %xmm7, %xmm7
13802        vpslld	$0x01, %xmm4, %xmm4
13803        vpsrldq	$12, %xmm0, %xmm2
13804        vpslldq	$4, %xmm0, %xmm0
13805        vpslldq	$4, %xmm1, %xmm1
13806        vpor	%xmm2, %xmm4, %xmm4
13807        vpor	%xmm0, %xmm7, %xmm7
13808        vpor	%xmm1, %xmm4, %xmm4
13809        # ghash_red
13810        vmovdqu	L_avx2_aes_gcm_mod2_128(%rip), %xmm2
13811        vpclmulqdq	$16, %xmm2, %xmm7, %xmm0
13812        vpshufd	$0x4e, %xmm7, %xmm1
13813        vpxor	%xmm0, %xmm1, %xmm1
13814        vpclmulqdq	$16, %xmm2, %xmm1, %xmm0
13815        vpshufd	$0x4e, %xmm1, %xmm1
13816        vpxor	%xmm0, %xmm1, %xmm1
13817        vpxor	%xmm1, %xmm4, %xmm4
13818        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm4, %xmm4
13819        #   Encrypt counter
13820        vmovdqu	(%rsi), %xmm15
13821        vpxor	%xmm4, %xmm15, %xmm15
13822        vaesenc	16(%rsi), %xmm15, %xmm15
13823        vaesenc	32(%rsi), %xmm15, %xmm15
13824        vaesenc	48(%rsi), %xmm15, %xmm15
13825        vaesenc	64(%rsi), %xmm15, %xmm15
13826        vaesenc	80(%rsi), %xmm15, %xmm15
13827        vaesenc	96(%rsi), %xmm15, %xmm15
13828        vaesenc	112(%rsi), %xmm15, %xmm15
13829        vaesenc	128(%rsi), %xmm15, %xmm15
13830        vaesenc	144(%rsi), %xmm15, %xmm15
13831        cmpl	$11, %r9d
13832        vmovdqu	160(%rsi), %xmm0
13833        jl	L_AES_GCM_decrypt_avx2_calc_iv_2_aesenc_avx_last
13834        vaesenc	%xmm0, %xmm15, %xmm15
13835        vaesenc	176(%rsi), %xmm15, %xmm15
13836        cmpl	$13, %r9d
13837        vmovdqu	192(%rsi), %xmm0
13838        jl	L_AES_GCM_decrypt_avx2_calc_iv_2_aesenc_avx_last
13839        vaesenc	%xmm0, %xmm15, %xmm15
13840        vaesenc	208(%rsi), %xmm15, %xmm15
13841        vmovdqu	224(%rsi), %xmm0
13842L_AES_GCM_decrypt_avx2_calc_iv_2_aesenc_avx_last:
13843        vaesenclast	%xmm0, %xmm15, %xmm15
13844        jmp	L_AES_GCM_decrypt_avx2_iv_done
13845L_AES_GCM_decrypt_avx2_iv_12:
13846        # # Calculate values when IV is 12 bytes
13847        # Set counter based on IV
13848        vmovdqu	L_avx2_aes_gcm_bswap_one(%rip), %xmm4
13849        vmovdqu	(%rsi), %xmm5
13850        vpblendd	$7, (%rax), %xmm4, %xmm4
13851        # H = Encrypt X(=0) and T = Encrypt counter
13852        vmovdqu	16(%rsi), %xmm7
13853        vpxor	%xmm5, %xmm4, %xmm15
13854        vaesenc	%xmm7, %xmm5, %xmm5
13855        vaesenc	%xmm7, %xmm15, %xmm15
13856        vmovdqu	32(%rsi), %xmm0
13857        vaesenc	%xmm0, %xmm5, %xmm5
13858        vaesenc	%xmm0, %xmm15, %xmm15
13859        vmovdqu	48(%rsi), %xmm0
13860        vaesenc	%xmm0, %xmm5, %xmm5
13861        vaesenc	%xmm0, %xmm15, %xmm15
13862        vmovdqu	64(%rsi), %xmm0
13863        vaesenc	%xmm0, %xmm5, %xmm5
13864        vaesenc	%xmm0, %xmm15, %xmm15
13865        vmovdqu	80(%rsi), %xmm0
13866        vaesenc	%xmm0, %xmm5, %xmm5
13867        vaesenc	%xmm0, %xmm15, %xmm15
13868        vmovdqu	96(%rsi), %xmm0
13869        vaesenc	%xmm0, %xmm5, %xmm5
13870        vaesenc	%xmm0, %xmm15, %xmm15
13871        vmovdqu	112(%rsi), %xmm0
13872        vaesenc	%xmm0, %xmm5, %xmm5
13873        vaesenc	%xmm0, %xmm15, %xmm15
13874        vmovdqu	128(%rsi), %xmm0
13875        vaesenc	%xmm0, %xmm5, %xmm5
13876        vaesenc	%xmm0, %xmm15, %xmm15
13877        vmovdqu	144(%rsi), %xmm0
13878        vaesenc	%xmm0, %xmm5, %xmm5
13879        vaesenc	%xmm0, %xmm15, %xmm15
13880        cmpl	$11, %r9d
13881        vmovdqu	160(%rsi), %xmm0
13882        jl	L_AES_GCM_decrypt_avx2_calc_iv_12_last
13883        vaesenc	%xmm0, %xmm5, %xmm5
13884        vaesenc	%xmm0, %xmm15, %xmm15
13885        vmovdqu	176(%rsi), %xmm0
13886        vaesenc	%xmm0, %xmm5, %xmm5
13887        vaesenc	%xmm0, %xmm15, %xmm15
13888        cmpl	$13, %r9d
13889        vmovdqu	192(%rsi), %xmm0
13890        jl	L_AES_GCM_decrypt_avx2_calc_iv_12_last
13891        vaesenc	%xmm0, %xmm5, %xmm5
13892        vaesenc	%xmm0, %xmm15, %xmm15
13893        vmovdqu	208(%rsi), %xmm0
13894        vaesenc	%xmm0, %xmm5, %xmm5
13895        vaesenc	%xmm0, %xmm15, %xmm15
13896        vmovdqu	224(%rsi), %xmm0
13897L_AES_GCM_decrypt_avx2_calc_iv_12_last:
13898        vaesenclast	%xmm0, %xmm5, %xmm5
13899        vaesenclast	%xmm0, %xmm15, %xmm15
13900        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm5, %xmm5
13901L_AES_GCM_decrypt_avx2_iv_done:
13902        # Additional authentication data
13903        movl	%r11d, %edx
13904        cmpl	$0x00, %edx
13905        je	L_AES_GCM_decrypt_avx2_calc_aad_done
13906        xorl	%ecx, %ecx
13907        cmpl	$16, %edx
13908        jl	L_AES_GCM_decrypt_avx2_calc_aad_lt16
13909        andl	$0xfffffff0, %edx
13910L_AES_GCM_decrypt_avx2_calc_aad_16_loop:
13911        vmovdqu	(%r12,%rcx,1), %xmm0
13912        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0
13913        vpxor	%xmm0, %xmm6, %xmm6
13914        # ghash_gfmul_avx
13915        vpclmulqdq	$16, %xmm6, %xmm5, %xmm2
13916        vpclmulqdq	$0x01, %xmm6, %xmm5, %xmm1
13917        vpclmulqdq	$0x00, %xmm6, %xmm5, %xmm0
13918        vpclmulqdq	$0x11, %xmm6, %xmm5, %xmm3
13919        vpxor	%xmm1, %xmm2, %xmm2
13920        vpslldq	$8, %xmm2, %xmm1
13921        vpsrldq	$8, %xmm2, %xmm2
13922        vpxor	%xmm1, %xmm0, %xmm7
13923        vpxor	%xmm2, %xmm3, %xmm6
13924        # ghash_mid
13925        vpsrld	$31, %xmm7, %xmm0
13926        vpsrld	$31, %xmm6, %xmm1
13927        vpslld	$0x01, %xmm7, %xmm7
13928        vpslld	$0x01, %xmm6, %xmm6
13929        vpsrldq	$12, %xmm0, %xmm2
13930        vpslldq	$4, %xmm0, %xmm0
13931        vpslldq	$4, %xmm1, %xmm1
13932        vpor	%xmm2, %xmm6, %xmm6
13933        vpor	%xmm0, %xmm7, %xmm7
13934        vpor	%xmm1, %xmm6, %xmm6
13935        # ghash_red
13936        vmovdqu	L_avx2_aes_gcm_mod2_128(%rip), %xmm2
13937        vpclmulqdq	$16, %xmm2, %xmm7, %xmm0
13938        vpshufd	$0x4e, %xmm7, %xmm1
13939        vpxor	%xmm0, %xmm1, %xmm1
13940        vpclmulqdq	$16, %xmm2, %xmm1, %xmm0
13941        vpshufd	$0x4e, %xmm1, %xmm1
13942        vpxor	%xmm0, %xmm1, %xmm1
13943        vpxor	%xmm1, %xmm6, %xmm6
13944        addl	$16, %ecx
13945        cmpl	%edx, %ecx
13946        jl	L_AES_GCM_decrypt_avx2_calc_aad_16_loop
13947        movl	%r11d, %edx
13948        cmpl	%edx, %ecx
13949        je	L_AES_GCM_decrypt_avx2_calc_aad_done
13950L_AES_GCM_decrypt_avx2_calc_aad_lt16:
13951        vpxor	%xmm0, %xmm0, %xmm0
13952        xorl	%ebx, %ebx
13953        vmovdqu	%xmm0, (%rsp)
13954L_AES_GCM_decrypt_avx2_calc_aad_loop:
13955        movzbl	(%r12,%rcx,1), %r13d
13956        movb	%r13b, (%rsp,%rbx,1)
13957        incl	%ecx
13958        incl	%ebx
13959        cmpl	%edx, %ecx
13960        jl	L_AES_GCM_decrypt_avx2_calc_aad_loop
13961        vmovdqu	(%rsp), %xmm0
13962        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0
13963        vpxor	%xmm0, %xmm6, %xmm6
13964        # ghash_gfmul_avx
13965        vpclmulqdq	$16, %xmm6, %xmm5, %xmm2
13966        vpclmulqdq	$0x01, %xmm6, %xmm5, %xmm1
13967        vpclmulqdq	$0x00, %xmm6, %xmm5, %xmm0
13968        vpclmulqdq	$0x11, %xmm6, %xmm5, %xmm3
13969        vpxor	%xmm1, %xmm2, %xmm2
13970        vpslldq	$8, %xmm2, %xmm1
13971        vpsrldq	$8, %xmm2, %xmm2
13972        vpxor	%xmm1, %xmm0, %xmm7
13973        vpxor	%xmm2, %xmm3, %xmm6
13974        # ghash_mid
13975        vpsrld	$31, %xmm7, %xmm0
13976        vpsrld	$31, %xmm6, %xmm1
13977        vpslld	$0x01, %xmm7, %xmm7
13978        vpslld	$0x01, %xmm6, %xmm6
13979        vpsrldq	$12, %xmm0, %xmm2
13980        vpslldq	$4, %xmm0, %xmm0
13981        vpslldq	$4, %xmm1, %xmm1
13982        vpor	%xmm2, %xmm6, %xmm6
13983        vpor	%xmm0, %xmm7, %xmm7
13984        vpor	%xmm1, %xmm6, %xmm6
13985        # ghash_red
13986        vmovdqu	L_avx2_aes_gcm_mod2_128(%rip), %xmm2
13987        vpclmulqdq	$16, %xmm2, %xmm7, %xmm0
13988        vpshufd	$0x4e, %xmm7, %xmm1
13989        vpxor	%xmm0, %xmm1, %xmm1
13990        vpclmulqdq	$16, %xmm2, %xmm1, %xmm0
13991        vpshufd	$0x4e, %xmm1, %xmm1
13992        vpxor	%xmm0, %xmm1, %xmm1
13993        vpxor	%xmm1, %xmm6, %xmm6
13994L_AES_GCM_decrypt_avx2_calc_aad_done:
13995        # Calculate counter and H
13996        vpsrlq	$63, %xmm5, %xmm1
13997        vpsllq	$0x01, %xmm5, %xmm0
13998        vpslldq	$8, %xmm1, %xmm1
13999        vpor	%xmm1, %xmm0, %xmm0
14000        vpshufd	$0xff, %xmm5, %xmm5
14001        vpsrad	$31, %xmm5, %xmm5
14002        vpshufb	L_avx2_aes_gcm_bswap_epi64(%rip), %xmm4, %xmm4
14003        vpand	L_avx2_aes_gcm_mod2_128(%rip), %xmm5, %xmm5
14004        vpaddd	L_avx2_aes_gcm_one(%rip), %xmm4, %xmm4
14005        vpxor	%xmm0, %xmm5, %xmm5
14006        xorl	%ebx, %ebx
14007        cmpl	$0x80, %r10d
14008        movl	%r10d, %r13d
14009        jl	L_AES_GCM_decrypt_avx2_done_128
14010        andl	$0xffffff80, %r13d
14011        vmovdqu	%xmm4, 128(%rsp)
14012        vmovdqu	%xmm15, 144(%rsp)
14013        vmovdqu	L_avx2_aes_gcm_mod2_128(%rip), %xmm3
14014        # H ^ 1 and H ^ 2
14015        vpclmulqdq	$0x00, %xmm5, %xmm5, %xmm9
14016        vpclmulqdq	$0x11, %xmm5, %xmm5, %xmm10
14017        vpclmulqdq	$16, %xmm3, %xmm9, %xmm8
14018        vpshufd	$0x4e, %xmm9, %xmm9
14019        vpxor	%xmm8, %xmm9, %xmm9
14020        vpclmulqdq	$16, %xmm3, %xmm9, %xmm8
14021        vpshufd	$0x4e, %xmm9, %xmm9
14022        vpxor	%xmm8, %xmm9, %xmm9
14023        vpxor	%xmm9, %xmm10, %xmm0
14024        vmovdqu	%xmm5, (%rsp)
14025        vmovdqu	%xmm0, 16(%rsp)
14026        # H ^ 3 and H ^ 4
14027        vpclmulqdq	$16, %xmm5, %xmm0, %xmm11
14028        vpclmulqdq	$0x01, %xmm5, %xmm0, %xmm10
14029        vpclmulqdq	$0x00, %xmm5, %xmm0, %xmm9
14030        vpclmulqdq	$0x11, %xmm5, %xmm0, %xmm12
14031        vpclmulqdq	$0x00, %xmm0, %xmm0, %xmm13
14032        vpclmulqdq	$0x11, %xmm0, %xmm0, %xmm14
14033        vpxor	%xmm10, %xmm11, %xmm11
14034        vpslldq	$8, %xmm11, %xmm10
14035        vpsrldq	$8, %xmm11, %xmm11
14036        vpxor	%xmm9, %xmm10, %xmm10
14037        vpclmulqdq	$16, %xmm3, %xmm13, %xmm8
14038        vpclmulqdq	$16, %xmm3, %xmm10, %xmm9
14039        vpshufd	$0x4e, %xmm10, %xmm10
14040        vpshufd	$0x4e, %xmm13, %xmm13
14041        vpxor	%xmm9, %xmm10, %xmm10
14042        vpxor	%xmm8, %xmm13, %xmm13
14043        vpclmulqdq	$16, %xmm3, %xmm10, %xmm9
14044        vpclmulqdq	$16, %xmm3, %xmm13, %xmm8
14045        vpshufd	$0x4e, %xmm10, %xmm10
14046        vpshufd	$0x4e, %xmm13, %xmm13
14047        vpxor	%xmm11, %xmm12, %xmm12
14048        vpxor	%xmm8, %xmm13, %xmm13
14049        vpxor	%xmm12, %xmm10, %xmm10
14050        vpxor	%xmm14, %xmm13, %xmm2
14051        vpxor	%xmm9, %xmm10, %xmm1
14052        vmovdqu	%xmm1, 32(%rsp)
14053        vmovdqu	%xmm2, 48(%rsp)
14054        # H ^ 5 and H ^ 6
14055        vpclmulqdq	$16, %xmm0, %xmm1, %xmm11
14056        vpclmulqdq	$0x01, %xmm0, %xmm1, %xmm10
14057        vpclmulqdq	$0x00, %xmm0, %xmm1, %xmm9
14058        vpclmulqdq	$0x11, %xmm0, %xmm1, %xmm12
14059        vpclmulqdq	$0x00, %xmm1, %xmm1, %xmm13
14060        vpclmulqdq	$0x11, %xmm1, %xmm1, %xmm14
14061        vpxor	%xmm10, %xmm11, %xmm11
14062        vpslldq	$8, %xmm11, %xmm10
14063        vpsrldq	$8, %xmm11, %xmm11
14064        vpxor	%xmm9, %xmm10, %xmm10
14065        vpclmulqdq	$16, %xmm3, %xmm13, %xmm8
14066        vpclmulqdq	$16, %xmm3, %xmm10, %xmm9
14067        vpshufd	$0x4e, %xmm10, %xmm10
14068        vpshufd	$0x4e, %xmm13, %xmm13
14069        vpxor	%xmm9, %xmm10, %xmm10
14070        vpxor	%xmm8, %xmm13, %xmm13
14071        vpclmulqdq	$16, %xmm3, %xmm10, %xmm9
14072        vpclmulqdq	$16, %xmm3, %xmm13, %xmm8
14073        vpshufd	$0x4e, %xmm10, %xmm10
14074        vpshufd	$0x4e, %xmm13, %xmm13
14075        vpxor	%xmm11, %xmm12, %xmm12
14076        vpxor	%xmm8, %xmm13, %xmm13
14077        vpxor	%xmm12, %xmm10, %xmm10
14078        vpxor	%xmm14, %xmm13, %xmm0
14079        vpxor	%xmm9, %xmm10, %xmm7
14080        vmovdqu	%xmm7, 64(%rsp)
14081        vmovdqu	%xmm0, 80(%rsp)
14082        # H ^ 7 and H ^ 8
14083        vpclmulqdq	$16, %xmm1, %xmm2, %xmm11
14084        vpclmulqdq	$0x01, %xmm1, %xmm2, %xmm10
14085        vpclmulqdq	$0x00, %xmm1, %xmm2, %xmm9
14086        vpclmulqdq	$0x11, %xmm1, %xmm2, %xmm12
14087        vpclmulqdq	$0x00, %xmm2, %xmm2, %xmm13
14088        vpclmulqdq	$0x11, %xmm2, %xmm2, %xmm14
14089        vpxor	%xmm10, %xmm11, %xmm11
14090        vpslldq	$8, %xmm11, %xmm10
14091        vpsrldq	$8, %xmm11, %xmm11
14092        vpxor	%xmm9, %xmm10, %xmm10
14093        vpclmulqdq	$16, %xmm3, %xmm13, %xmm8
14094        vpclmulqdq	$16, %xmm3, %xmm10, %xmm9
14095        vpshufd	$0x4e, %xmm10, %xmm10
14096        vpshufd	$0x4e, %xmm13, %xmm13
14097        vpxor	%xmm9, %xmm10, %xmm10
14098        vpxor	%xmm8, %xmm13, %xmm13
14099        vpclmulqdq	$16, %xmm3, %xmm10, %xmm9
14100        vpclmulqdq	$16, %xmm3, %xmm13, %xmm8
14101        vpshufd	$0x4e, %xmm10, %xmm10
14102        vpshufd	$0x4e, %xmm13, %xmm13
14103        vpxor	%xmm11, %xmm12, %xmm12
14104        vpxor	%xmm8, %xmm13, %xmm13
14105        vpxor	%xmm12, %xmm10, %xmm10
14106        vpxor	%xmm14, %xmm13, %xmm0
14107        vpxor	%xmm9, %xmm10, %xmm7
14108        vmovdqu	%xmm7, 96(%rsp)
14109        vmovdqu	%xmm0, 112(%rsp)
14110L_AES_GCM_decrypt_avx2_ghash_128:
14111        # aesenc_128_ghash
14112        leaq	(%rdi,%rbx,1), %rcx
14113        leaq	(%r8,%rbx,1), %rdx
14114        # aesenc_ctr
14115        vmovdqu	128(%rsp), %xmm0
14116        vmovdqu	L_avx2_aes_gcm_bswap_epi64(%rip), %xmm1
14117        vpaddd	L_avx2_aes_gcm_one(%rip), %xmm0, %xmm9
14118        vpshufb	%xmm1, %xmm0, %xmm8
14119        vpaddd	L_avx2_aes_gcm_two(%rip), %xmm0, %xmm10
14120        vpshufb	%xmm1, %xmm9, %xmm9
14121        vpaddd	L_avx2_aes_gcm_three(%rip), %xmm0, %xmm11
14122        vpshufb	%xmm1, %xmm10, %xmm10
14123        vpaddd	L_avx2_aes_gcm_four(%rip), %xmm0, %xmm12
14124        vpshufb	%xmm1, %xmm11, %xmm11
14125        vpaddd	L_avx2_aes_gcm_five(%rip), %xmm0, %xmm13
14126        vpshufb	%xmm1, %xmm12, %xmm12
14127        vpaddd	L_avx2_aes_gcm_six(%rip), %xmm0, %xmm14
14128        vpshufb	%xmm1, %xmm13, %xmm13
14129        vpaddd	L_avx2_aes_gcm_seven(%rip), %xmm0, %xmm15
14130        vpshufb	%xmm1, %xmm14, %xmm14
14131        vpaddd	L_avx2_aes_gcm_eight(%rip), %xmm0, %xmm0
14132        vpshufb	%xmm1, %xmm15, %xmm15
14133        # aesenc_xor
14134        vmovdqu	(%rsi), %xmm7
14135        vmovdqu	%xmm0, 128(%rsp)
14136        vpxor	%xmm7, %xmm8, %xmm8
14137        vpxor	%xmm7, %xmm9, %xmm9
14138        vpxor	%xmm7, %xmm10, %xmm10
14139        vpxor	%xmm7, %xmm11, %xmm11
14140        vpxor	%xmm7, %xmm12, %xmm12
14141        vpxor	%xmm7, %xmm13, %xmm13
14142        vpxor	%xmm7, %xmm14, %xmm14
14143        vpxor	%xmm7, %xmm15, %xmm15
14144        # aesenc_pclmul_1
14145        vmovdqu	(%rcx), %xmm1
14146        vmovdqu	16(%rsi), %xmm0
14147        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm1, %xmm1
14148        vmovdqu	112(%rsp), %xmm2
14149        vpxor	%xmm6, %xmm1, %xmm1
14150        vpclmulqdq	$16, %xmm2, %xmm1, %xmm5
14151        vpclmulqdq	$0x01, %xmm2, %xmm1, %xmm3
14152        vpclmulqdq	$0x00, %xmm2, %xmm1, %xmm6
14153        vpclmulqdq	$0x11, %xmm2, %xmm1, %xmm7
14154        vaesenc	%xmm0, %xmm8, %xmm8
14155        vaesenc	%xmm0, %xmm9, %xmm9
14156        vaesenc	%xmm0, %xmm10, %xmm10
14157        vaesenc	%xmm0, %xmm11, %xmm11
14158        vaesenc	%xmm0, %xmm12, %xmm12
14159        vaesenc	%xmm0, %xmm13, %xmm13
14160        vaesenc	%xmm0, %xmm14, %xmm14
14161        vaesenc	%xmm0, %xmm15, %xmm15
14162        # aesenc_pclmul_2
14163        vmovdqu	16(%rcx), %xmm1
14164        vmovdqu	96(%rsp), %xmm0
14165        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm1, %xmm1
14166        vpxor	%xmm3, %xmm5, %xmm5
14167        vpclmulqdq	$16, %xmm0, %xmm1, %xmm2
14168        vpclmulqdq	$0x01, %xmm0, %xmm1, %xmm3
14169        vpclmulqdq	$0x00, %xmm0, %xmm1, %xmm4
14170        vpclmulqdq	$0x11, %xmm0, %xmm1, %xmm1
14171        vmovdqu	32(%rsi), %xmm0
14172        vpxor	%xmm1, %xmm7, %xmm7
14173        vaesenc	%xmm0, %xmm8, %xmm8
14174        vaesenc	%xmm0, %xmm9, %xmm9
14175        vaesenc	%xmm0, %xmm10, %xmm10
14176        vaesenc	%xmm0, %xmm11, %xmm11
14177        vaesenc	%xmm0, %xmm12, %xmm12
14178        vaesenc	%xmm0, %xmm13, %xmm13
14179        vaesenc	%xmm0, %xmm14, %xmm14
14180        vaesenc	%xmm0, %xmm15, %xmm15
14181        # aesenc_pclmul_n
14182        vmovdqu	32(%rcx), %xmm1
14183        vmovdqu	80(%rsp), %xmm0
14184        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm1, %xmm1
14185        vpxor	%xmm2, %xmm5, %xmm5
14186        vpclmulqdq	$16, %xmm0, %xmm1, %xmm2
14187        vpxor	%xmm3, %xmm5, %xmm5
14188        vpclmulqdq	$0x01, %xmm0, %xmm1, %xmm3
14189        vpxor	%xmm4, %xmm6, %xmm6
14190        vpclmulqdq	$0x00, %xmm0, %xmm1, %xmm4
14191        vpclmulqdq	$0x11, %xmm0, %xmm1, %xmm1
14192        vmovdqu	48(%rsi), %xmm0
14193        vpxor	%xmm1, %xmm7, %xmm7
14194        vaesenc	%xmm0, %xmm8, %xmm8
14195        vaesenc	%xmm0, %xmm9, %xmm9
14196        vaesenc	%xmm0, %xmm10, %xmm10
14197        vaesenc	%xmm0, %xmm11, %xmm11
14198        vaesenc	%xmm0, %xmm12, %xmm12
14199        vaesenc	%xmm0, %xmm13, %xmm13
14200        vaesenc	%xmm0, %xmm14, %xmm14
14201        vaesenc	%xmm0, %xmm15, %xmm15
14202        # aesenc_pclmul_n
14203        vmovdqu	48(%rcx), %xmm1
14204        vmovdqu	64(%rsp), %xmm0
14205        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm1, %xmm1
14206        vpxor	%xmm2, %xmm5, %xmm5
14207        vpclmulqdq	$16, %xmm0, %xmm1, %xmm2
14208        vpxor	%xmm3, %xmm5, %xmm5
14209        vpclmulqdq	$0x01, %xmm0, %xmm1, %xmm3
14210        vpxor	%xmm4, %xmm6, %xmm6
14211        vpclmulqdq	$0x00, %xmm0, %xmm1, %xmm4
14212        vpclmulqdq	$0x11, %xmm0, %xmm1, %xmm1
14213        vmovdqu	64(%rsi), %xmm0
14214        vpxor	%xmm1, %xmm7, %xmm7
14215        vaesenc	%xmm0, %xmm8, %xmm8
14216        vaesenc	%xmm0, %xmm9, %xmm9
14217        vaesenc	%xmm0, %xmm10, %xmm10
14218        vaesenc	%xmm0, %xmm11, %xmm11
14219        vaesenc	%xmm0, %xmm12, %xmm12
14220        vaesenc	%xmm0, %xmm13, %xmm13
14221        vaesenc	%xmm0, %xmm14, %xmm14
14222        vaesenc	%xmm0, %xmm15, %xmm15
14223        # aesenc_pclmul_n
14224        vmovdqu	64(%rcx), %xmm1
14225        vmovdqu	48(%rsp), %xmm0
14226        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm1, %xmm1
14227        vpxor	%xmm2, %xmm5, %xmm5
14228        vpclmulqdq	$16, %xmm0, %xmm1, %xmm2
14229        vpxor	%xmm3, %xmm5, %xmm5
14230        vpclmulqdq	$0x01, %xmm0, %xmm1, %xmm3
14231        vpxor	%xmm4, %xmm6, %xmm6
14232        vpclmulqdq	$0x00, %xmm0, %xmm1, %xmm4
14233        vpclmulqdq	$0x11, %xmm0, %xmm1, %xmm1
14234        vmovdqu	80(%rsi), %xmm0
14235        vpxor	%xmm1, %xmm7, %xmm7
14236        vaesenc	%xmm0, %xmm8, %xmm8
14237        vaesenc	%xmm0, %xmm9, %xmm9
14238        vaesenc	%xmm0, %xmm10, %xmm10
14239        vaesenc	%xmm0, %xmm11, %xmm11
14240        vaesenc	%xmm0, %xmm12, %xmm12
14241        vaesenc	%xmm0, %xmm13, %xmm13
14242        vaesenc	%xmm0, %xmm14, %xmm14
14243        vaesenc	%xmm0, %xmm15, %xmm15
14244        # aesenc_pclmul_n
14245        vmovdqu	80(%rcx), %xmm1
14246        vmovdqu	32(%rsp), %xmm0
14247        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm1, %xmm1
14248        vpxor	%xmm2, %xmm5, %xmm5
14249        vpclmulqdq	$16, %xmm0, %xmm1, %xmm2
14250        vpxor	%xmm3, %xmm5, %xmm5
14251        vpclmulqdq	$0x01, %xmm0, %xmm1, %xmm3
14252        vpxor	%xmm4, %xmm6, %xmm6
14253        vpclmulqdq	$0x00, %xmm0, %xmm1, %xmm4
14254        vpclmulqdq	$0x11, %xmm0, %xmm1, %xmm1
14255        vmovdqu	96(%rsi), %xmm0
14256        vpxor	%xmm1, %xmm7, %xmm7
14257        vaesenc	%xmm0, %xmm8, %xmm8
14258        vaesenc	%xmm0, %xmm9, %xmm9
14259        vaesenc	%xmm0, %xmm10, %xmm10
14260        vaesenc	%xmm0, %xmm11, %xmm11
14261        vaesenc	%xmm0, %xmm12, %xmm12
14262        vaesenc	%xmm0, %xmm13, %xmm13
14263        vaesenc	%xmm0, %xmm14, %xmm14
14264        vaesenc	%xmm0, %xmm15, %xmm15
14265        # aesenc_pclmul_n
14266        vmovdqu	96(%rcx), %xmm1
14267        vmovdqu	16(%rsp), %xmm0
14268        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm1, %xmm1
14269        vpxor	%xmm2, %xmm5, %xmm5
14270        vpclmulqdq	$16, %xmm0, %xmm1, %xmm2
14271        vpxor	%xmm3, %xmm5, %xmm5
14272        vpclmulqdq	$0x01, %xmm0, %xmm1, %xmm3
14273        vpxor	%xmm4, %xmm6, %xmm6
14274        vpclmulqdq	$0x00, %xmm0, %xmm1, %xmm4
14275        vpclmulqdq	$0x11, %xmm0, %xmm1, %xmm1
14276        vmovdqu	112(%rsi), %xmm0
14277        vpxor	%xmm1, %xmm7, %xmm7
14278        vaesenc	%xmm0, %xmm8, %xmm8
14279        vaesenc	%xmm0, %xmm9, %xmm9
14280        vaesenc	%xmm0, %xmm10, %xmm10
14281        vaesenc	%xmm0, %xmm11, %xmm11
14282        vaesenc	%xmm0, %xmm12, %xmm12
14283        vaesenc	%xmm0, %xmm13, %xmm13
14284        vaesenc	%xmm0, %xmm14, %xmm14
14285        vaesenc	%xmm0, %xmm15, %xmm15
14286        # aesenc_pclmul_n
14287        vmovdqu	112(%rcx), %xmm1
14288        vmovdqu	(%rsp), %xmm0
14289        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm1, %xmm1
14290        vpxor	%xmm2, %xmm5, %xmm5
14291        vpclmulqdq	$16, %xmm0, %xmm1, %xmm2
14292        vpxor	%xmm3, %xmm5, %xmm5
14293        vpclmulqdq	$0x01, %xmm0, %xmm1, %xmm3
14294        vpxor	%xmm4, %xmm6, %xmm6
14295        vpclmulqdq	$0x00, %xmm0, %xmm1, %xmm4
14296        vpclmulqdq	$0x11, %xmm0, %xmm1, %xmm1
14297        vmovdqu	128(%rsi), %xmm0
14298        vpxor	%xmm1, %xmm7, %xmm7
14299        vaesenc	%xmm0, %xmm8, %xmm8
14300        vaesenc	%xmm0, %xmm9, %xmm9
14301        vaesenc	%xmm0, %xmm10, %xmm10
14302        vaesenc	%xmm0, %xmm11, %xmm11
14303        vaesenc	%xmm0, %xmm12, %xmm12
14304        vaesenc	%xmm0, %xmm13, %xmm13
14305        vaesenc	%xmm0, %xmm14, %xmm14
14306        vaesenc	%xmm0, %xmm15, %xmm15
14307        # aesenc_pclmul_l
14308        vpxor	%xmm2, %xmm5, %xmm5
14309        vpxor	%xmm4, %xmm6, %xmm6
14310        vpxor	%xmm3, %xmm5, %xmm5
14311        vpslldq	$8, %xmm5, %xmm1
14312        vpsrldq	$8, %xmm5, %xmm5
14313        vmovdqu	144(%rsi), %xmm4
14314        vmovdqu	L_avx2_aes_gcm_mod2_128(%rip), %xmm0
14315        vaesenc	%xmm4, %xmm8, %xmm8
14316        vpxor	%xmm1, %xmm6, %xmm6
14317        vpxor	%xmm5, %xmm7, %xmm7
14318        vpclmulqdq	$16, %xmm0, %xmm6, %xmm3
14319        vaesenc	%xmm4, %xmm9, %xmm9
14320        vaesenc	%xmm4, %xmm10, %xmm10
14321        vaesenc	%xmm4, %xmm11, %xmm11
14322        vpshufd	$0x4e, %xmm6, %xmm6
14323        vpxor	%xmm3, %xmm6, %xmm6
14324        vpclmulqdq	$16, %xmm0, %xmm6, %xmm3
14325        vaesenc	%xmm4, %xmm12, %xmm12
14326        vaesenc	%xmm4, %xmm13, %xmm13
14327        vaesenc	%xmm4, %xmm14, %xmm14
14328        vpshufd	$0x4e, %xmm6, %xmm6
14329        vpxor	%xmm3, %xmm6, %xmm6
14330        vpxor	%xmm7, %xmm6, %xmm6
14331        vaesenc	%xmm4, %xmm15, %xmm15
14332        cmpl	$11, %r9d
14333        vmovdqu	160(%rsi), %xmm7
14334        jl	L_AES_GCM_decrypt_avx2_aesenc_128_ghash_avx_done
14335        vaesenc	%xmm7, %xmm8, %xmm8
14336        vaesenc	%xmm7, %xmm9, %xmm9
14337        vaesenc	%xmm7, %xmm10, %xmm10
14338        vaesenc	%xmm7, %xmm11, %xmm11
14339        vaesenc	%xmm7, %xmm12, %xmm12
14340        vaesenc	%xmm7, %xmm13, %xmm13
14341        vaesenc	%xmm7, %xmm14, %xmm14
14342        vaesenc	%xmm7, %xmm15, %xmm15
14343        vmovdqu	176(%rsi), %xmm7
14344        vaesenc	%xmm7, %xmm8, %xmm8
14345        vaesenc	%xmm7, %xmm9, %xmm9
14346        vaesenc	%xmm7, %xmm10, %xmm10
14347        vaesenc	%xmm7, %xmm11, %xmm11
14348        vaesenc	%xmm7, %xmm12, %xmm12
14349        vaesenc	%xmm7, %xmm13, %xmm13
14350        vaesenc	%xmm7, %xmm14, %xmm14
14351        vaesenc	%xmm7, %xmm15, %xmm15
14352        cmpl	$13, %r9d
14353        vmovdqu	192(%rsi), %xmm7
14354        jl	L_AES_GCM_decrypt_avx2_aesenc_128_ghash_avx_done
14355        vaesenc	%xmm7, %xmm8, %xmm8
14356        vaesenc	%xmm7, %xmm9, %xmm9
14357        vaesenc	%xmm7, %xmm10, %xmm10
14358        vaesenc	%xmm7, %xmm11, %xmm11
14359        vaesenc	%xmm7, %xmm12, %xmm12
14360        vaesenc	%xmm7, %xmm13, %xmm13
14361        vaesenc	%xmm7, %xmm14, %xmm14
14362        vaesenc	%xmm7, %xmm15, %xmm15
14363        vmovdqu	208(%rsi), %xmm7
14364        vaesenc	%xmm7, %xmm8, %xmm8
14365        vaesenc	%xmm7, %xmm9, %xmm9
14366        vaesenc	%xmm7, %xmm10, %xmm10
14367        vaesenc	%xmm7, %xmm11, %xmm11
14368        vaesenc	%xmm7, %xmm12, %xmm12
14369        vaesenc	%xmm7, %xmm13, %xmm13
14370        vaesenc	%xmm7, %xmm14, %xmm14
14371        vaesenc	%xmm7, %xmm15, %xmm15
14372        vmovdqu	224(%rsi), %xmm7
14373L_AES_GCM_decrypt_avx2_aesenc_128_ghash_avx_done:
14374        # aesenc_last
14375        vaesenclast	%xmm7, %xmm8, %xmm8
14376        vaesenclast	%xmm7, %xmm9, %xmm9
14377        vaesenclast	%xmm7, %xmm10, %xmm10
14378        vaesenclast	%xmm7, %xmm11, %xmm11
14379        vmovdqu	(%rcx), %xmm0
14380        vmovdqu	16(%rcx), %xmm1
14381        vmovdqu	32(%rcx), %xmm2
14382        vmovdqu	48(%rcx), %xmm3
14383        vpxor	%xmm0, %xmm8, %xmm8
14384        vpxor	%xmm1, %xmm9, %xmm9
14385        vpxor	%xmm2, %xmm10, %xmm10
14386        vpxor	%xmm3, %xmm11, %xmm11
14387        vmovdqu	%xmm8, (%rdx)
14388        vmovdqu	%xmm9, 16(%rdx)
14389        vmovdqu	%xmm10, 32(%rdx)
14390        vmovdqu	%xmm11, 48(%rdx)
14391        vaesenclast	%xmm7, %xmm12, %xmm12
14392        vaesenclast	%xmm7, %xmm13, %xmm13
14393        vaesenclast	%xmm7, %xmm14, %xmm14
14394        vaesenclast	%xmm7, %xmm15, %xmm15
14395        vmovdqu	64(%rcx), %xmm0
14396        vmovdqu	80(%rcx), %xmm1
14397        vmovdqu	96(%rcx), %xmm2
14398        vmovdqu	112(%rcx), %xmm3
14399        vpxor	%xmm0, %xmm12, %xmm12
14400        vpxor	%xmm1, %xmm13, %xmm13
14401        vpxor	%xmm2, %xmm14, %xmm14
14402        vpxor	%xmm3, %xmm15, %xmm15
14403        vmovdqu	%xmm12, 64(%rdx)
14404        vmovdqu	%xmm13, 80(%rdx)
14405        vmovdqu	%xmm14, 96(%rdx)
14406        vmovdqu	%xmm15, 112(%rdx)
14407        # aesenc_128_ghash - end
14408        addl	$0x80, %ebx
14409        cmpl	%r13d, %ebx
14410        jl	L_AES_GCM_decrypt_avx2_ghash_128
14411        vmovdqu	(%rsp), %xmm5
14412        vmovdqu	128(%rsp), %xmm4
14413        vmovdqu	144(%rsp), %xmm15
14414L_AES_GCM_decrypt_avx2_done_128:
14415        cmpl	%r10d, %ebx
14416        jge	L_AES_GCM_decrypt_avx2_done_dec
14417        movl	%r10d, %r13d
14418        andl	$0xfffffff0, %r13d
14419        cmpl	%r13d, %ebx
14420        jge	L_AES_GCM_decrypt_avx2_last_block_done
14421L_AES_GCM_decrypt_avx2_last_block_start:
14422        vmovdqu	(%rdi,%rbx,1), %xmm11
14423        vpshufb	L_avx2_aes_gcm_bswap_epi64(%rip), %xmm4, %xmm10
14424        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm11, %xmm12
14425        vpaddd	L_avx2_aes_gcm_one(%rip), %xmm4, %xmm4
14426        vpxor	%xmm6, %xmm12, %xmm12
14427        # aesenc_gfmul_sb
14428        vpclmulqdq	$0x01, %xmm5, %xmm12, %xmm2
14429        vpclmulqdq	$16, %xmm5, %xmm12, %xmm3
14430        vpclmulqdq	$0x00, %xmm5, %xmm12, %xmm1
14431        vpclmulqdq	$0x11, %xmm5, %xmm12, %xmm8
14432        vpxor	(%rsi), %xmm10, %xmm10
14433        vaesenc	16(%rsi), %xmm10, %xmm10
14434        vpxor	%xmm2, %xmm3, %xmm3
14435        vpslldq	$8, %xmm3, %xmm2
14436        vpsrldq	$8, %xmm3, %xmm3
14437        vaesenc	32(%rsi), %xmm10, %xmm10
14438        vpxor	%xmm1, %xmm2, %xmm2
14439        vpclmulqdq	$16, L_avx2_aes_gcm_mod2_128(%rip), %xmm2, %xmm1
14440        vaesenc	48(%rsi), %xmm10, %xmm10
14441        vaesenc	64(%rsi), %xmm10, %xmm10
14442        vaesenc	80(%rsi), %xmm10, %xmm10
14443        vpshufd	$0x4e, %xmm2, %xmm2
14444        vpxor	%xmm1, %xmm2, %xmm2
14445        vpclmulqdq	$16, L_avx2_aes_gcm_mod2_128(%rip), %xmm2, %xmm1
14446        vaesenc	96(%rsi), %xmm10, %xmm10
14447        vaesenc	112(%rsi), %xmm10, %xmm10
14448        vaesenc	128(%rsi), %xmm10, %xmm10
14449        vpshufd	$0x4e, %xmm2, %xmm2
14450        vaesenc	144(%rsi), %xmm10, %xmm10
14451        vpxor	%xmm3, %xmm8, %xmm8
14452        vpxor	%xmm8, %xmm2, %xmm2
14453        vmovdqu	160(%rsi), %xmm0
14454        cmpl	$11, %r9d
14455        jl	L_AES_GCM_decrypt_avx2_aesenc_gfmul_sb_last
14456        vaesenc	%xmm0, %xmm10, %xmm10
14457        vaesenc	176(%rsi), %xmm10, %xmm10
14458        vmovdqu	192(%rsi), %xmm0
14459        cmpl	$13, %r9d
14460        jl	L_AES_GCM_decrypt_avx2_aesenc_gfmul_sb_last
14461        vaesenc	%xmm0, %xmm10, %xmm10
14462        vaesenc	208(%rsi), %xmm10, %xmm10
14463        vmovdqu	224(%rsi), %xmm0
14464L_AES_GCM_decrypt_avx2_aesenc_gfmul_sb_last:
14465        vaesenclast	%xmm0, %xmm10, %xmm10
14466        vpxor	%xmm1, %xmm2, %xmm6
14467        vpxor	%xmm11, %xmm10, %xmm10
14468        vmovdqu	%xmm10, (%r8,%rbx,1)
14469        addl	$16, %ebx
14470        cmpl	%r13d, %ebx
14471        jl	L_AES_GCM_decrypt_avx2_last_block_start
14472L_AES_GCM_decrypt_avx2_last_block_done:
14473        movl	%r10d, %ecx
14474        movl	%r10d, %edx
14475        andl	$15, %ecx
14476        jz	L_AES_GCM_decrypt_avx2_done_dec
14477        # aesenc_last15_dec
14478        vpshufb	L_avx2_aes_gcm_bswap_epi64(%rip), %xmm4, %xmm4
14479        vpxor	(%rsi), %xmm4, %xmm4
14480        vaesenc	16(%rsi), %xmm4, %xmm4
14481        vaesenc	32(%rsi), %xmm4, %xmm4
14482        vaesenc	48(%rsi), %xmm4, %xmm4
14483        vaesenc	64(%rsi), %xmm4, %xmm4
14484        vaesenc	80(%rsi), %xmm4, %xmm4
14485        vaesenc	96(%rsi), %xmm4, %xmm4
14486        vaesenc	112(%rsi), %xmm4, %xmm4
14487        vaesenc	128(%rsi), %xmm4, %xmm4
14488        vaesenc	144(%rsi), %xmm4, %xmm4
14489        cmpl	$11, %r9d
14490        vmovdqu	160(%rsi), %xmm1
14491        jl	L_AES_GCM_decrypt_avx2_aesenc_last15_dec_avx_aesenc_avx_last
14492        vaesenc	%xmm1, %xmm4, %xmm4
14493        vaesenc	176(%rsi), %xmm4, %xmm4
14494        cmpl	$13, %r9d
14495        vmovdqu	192(%rsi), %xmm1
14496        jl	L_AES_GCM_decrypt_avx2_aesenc_last15_dec_avx_aesenc_avx_last
14497        vaesenc	%xmm1, %xmm4, %xmm4
14498        vaesenc	208(%rsi), %xmm4, %xmm4
14499        vmovdqu	224(%rsi), %xmm1
14500L_AES_GCM_decrypt_avx2_aesenc_last15_dec_avx_aesenc_avx_last:
14501        vaesenclast	%xmm1, %xmm4, %xmm4
14502        xorl	%ecx, %ecx
14503        vpxor	%xmm0, %xmm0, %xmm0
14504        vmovdqu	%xmm4, (%rsp)
14505        vmovdqu	%xmm0, 16(%rsp)
14506L_AES_GCM_decrypt_avx2_aesenc_last15_dec_avx_loop:
14507        movzbl	(%rdi,%rbx,1), %r13d
14508        movb	%r13b, 16(%rsp,%rcx,1)
14509        xorb	(%rsp,%rcx,1), %r13b
14510        movb	%r13b, (%r8,%rbx,1)
14511        incl	%ebx
14512        incl	%ecx
14513        cmpl	%edx, %ebx
14514        jl	L_AES_GCM_decrypt_avx2_aesenc_last15_dec_avx_loop
14515        vmovdqu	16(%rsp), %xmm4
14516        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm4, %xmm4
14517        vpxor	%xmm4, %xmm6, %xmm6
14518        # ghash_gfmul_red
14519        vpclmulqdq	$16, %xmm5, %xmm6, %xmm2
14520        vpclmulqdq	$0x01, %xmm5, %xmm6, %xmm1
14521        vpclmulqdq	$0x00, %xmm5, %xmm6, %xmm0
14522        vpxor	%xmm1, %xmm2, %xmm2
14523        vpslldq	$8, %xmm2, %xmm1
14524        vpsrldq	$8, %xmm2, %xmm2
14525        vpxor	%xmm0, %xmm1, %xmm1
14526        vpclmulqdq	$0x11, %xmm5, %xmm6, %xmm6
14527        vpclmulqdq	$16, L_avx2_aes_gcm_mod2_128(%rip), %xmm1, %xmm0
14528        vpshufd	$0x4e, %xmm1, %xmm1
14529        vpxor	%xmm0, %xmm1, %xmm1
14530        vpclmulqdq	$16, L_avx2_aes_gcm_mod2_128(%rip), %xmm1, %xmm0
14531        vpshufd	$0x4e, %xmm1, %xmm1
14532        vpxor	%xmm2, %xmm6, %xmm6
14533        vpxor	%xmm1, %xmm6, %xmm6
14534        vpxor	%xmm0, %xmm6, %xmm6
14535L_AES_GCM_decrypt_avx2_done_dec:
14536        # calc_tag
14537        shlq	$3, %r10
14538        shlq	$3, %r11
14539        vmovq	%r10, %xmm0
14540        vmovq	%r11, %xmm1
14541        vpunpcklqdq	%xmm1, %xmm0, %xmm0
14542        vpxor	%xmm6, %xmm0, %xmm0
14543        # ghash_gfmul_red
14544        vpclmulqdq	$16, %xmm5, %xmm0, %xmm4
14545        vpclmulqdq	$0x01, %xmm5, %xmm0, %xmm3
14546        vpclmulqdq	$0x00, %xmm5, %xmm0, %xmm2
14547        vpxor	%xmm3, %xmm4, %xmm4
14548        vpslldq	$8, %xmm4, %xmm3
14549        vpsrldq	$8, %xmm4, %xmm4
14550        vpxor	%xmm2, %xmm3, %xmm3
14551        vpclmulqdq	$0x11, %xmm5, %xmm0, %xmm0
14552        vpclmulqdq	$16, L_avx2_aes_gcm_mod2_128(%rip), %xmm3, %xmm2
14553        vpshufd	$0x4e, %xmm3, %xmm3
14554        vpxor	%xmm2, %xmm3, %xmm3
14555        vpclmulqdq	$16, L_avx2_aes_gcm_mod2_128(%rip), %xmm3, %xmm2
14556        vpshufd	$0x4e, %xmm3, %xmm3
14557        vpxor	%xmm4, %xmm0, %xmm0
14558        vpxor	%xmm3, %xmm0, %xmm0
14559        vpxor	%xmm2, %xmm0, %xmm0
14560        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0
14561        vpxor	%xmm15, %xmm0, %xmm0
14562        # cmp_tag
14563        cmpl	$16, %r15d
14564        je	L_AES_GCM_decrypt_avx2_cmp_tag_16
14565        xorq	%rdx, %rdx
14566        xorq	%rax, %rax
14567        vmovdqu	%xmm0, (%rsp)
14568L_AES_GCM_decrypt_avx2_cmp_tag_loop:
14569        movzbl	(%rsp,%rdx,1), %r13d
14570        xorb	(%r14,%rdx,1), %r13b
14571        orb	%r13b, %al
14572        incl	%edx
14573        cmpl	%r15d, %edx
14574        jne	L_AES_GCM_decrypt_avx2_cmp_tag_loop
14575        cmpb	$0x00, %al
14576        sete	%al
14577        jmp	L_AES_GCM_decrypt_avx2_cmp_tag_done
14578L_AES_GCM_decrypt_avx2_cmp_tag_16:
14579        vmovdqu	(%r14), %xmm1
14580        vpcmpeqb	%xmm1, %xmm0, %xmm0
14581        vpmovmskb	%xmm0, %rdx
14582        # %%edx == 0xFFFF then return 1 else => return 0
14583        xorl	%eax, %eax
14584        cmpl	$0xffff, %edx
14585        sete	%al
14586L_AES_GCM_decrypt_avx2_cmp_tag_done:
14587        movl	%eax, (%rbp)
14588        vzeroupper
14589        addq	$0xa8, %rsp
14590        popq	%rbp
14591        popq	%r15
14592        popq	%rbx
14593        popq	%r14
14594        popq	%r12
14595        popq	%r13
14596        repz retq
14597#ifndef __APPLE__
14598.size	AES_GCM_decrypt_avx2,.-AES_GCM_decrypt_avx2
14599#endif /* __APPLE__ */
14600#ifdef WOLFSSL_AESGCM_STREAM
14601#ifndef __APPLE__
14602.text
14603.globl	AES_GCM_init_avx2
14604.type	AES_GCM_init_avx2,@function
14605.align	16
14606AES_GCM_init_avx2:
14607#else
14608.section	__TEXT,__text
14609.globl	_AES_GCM_init_avx2
14610.p2align	4
14611_AES_GCM_init_avx2:
14612#endif /* __APPLE__ */
14613        pushq	%rbx
14614        pushq	%r12
14615        movq	%rdx, %r10
14616        movl	%ecx, %r11d
14617        movq	24(%rsp), %rax
14618        subq	$16, %rsp
14619        vpxor	%xmm4, %xmm4, %xmm4
14620        movl	%r11d, %edx
14621        cmpl	$12, %edx
14622        je	L_AES_GCM_init_avx2_iv_12
14623        # Calculate values when IV is not 12 bytes
14624        # H = Encrypt X(=0)
14625        vmovdqu	(%rdi), %xmm5
14626        vaesenc	16(%rdi), %xmm5, %xmm5
14627        vaesenc	32(%rdi), %xmm5, %xmm5
14628        vaesenc	48(%rdi), %xmm5, %xmm5
14629        vaesenc	64(%rdi), %xmm5, %xmm5
14630        vaesenc	80(%rdi), %xmm5, %xmm5
14631        vaesenc	96(%rdi), %xmm5, %xmm5
14632        vaesenc	112(%rdi), %xmm5, %xmm5
14633        vaesenc	128(%rdi), %xmm5, %xmm5
14634        vaesenc	144(%rdi), %xmm5, %xmm5
14635        cmpl	$11, %esi
14636        vmovdqu	160(%rdi), %xmm0
14637        jl	L_AES_GCM_init_avx2_calc_iv_1_aesenc_avx_last
14638        vaesenc	%xmm0, %xmm5, %xmm5
14639        vaesenc	176(%rdi), %xmm5, %xmm5
14640        cmpl	$13, %esi
14641        vmovdqu	192(%rdi), %xmm0
14642        jl	L_AES_GCM_init_avx2_calc_iv_1_aesenc_avx_last
14643        vaesenc	%xmm0, %xmm5, %xmm5
14644        vaesenc	208(%rdi), %xmm5, %xmm5
14645        vmovdqu	224(%rdi), %xmm0
14646L_AES_GCM_init_avx2_calc_iv_1_aesenc_avx_last:
14647        vaesenclast	%xmm0, %xmm5, %xmm5
14648        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm5, %xmm5
14649        # Calc counter
14650        # Initialization vector
14651        cmpl	$0x00, %edx
14652        movq	$0x00, %rcx
14653        je	L_AES_GCM_init_avx2_calc_iv_done
14654        cmpl	$16, %edx
14655        jl	L_AES_GCM_init_avx2_calc_iv_lt16
14656        andl	$0xfffffff0, %edx
14657L_AES_GCM_init_avx2_calc_iv_16_loop:
14658        vmovdqu	(%r10,%rcx,1), %xmm0
14659        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0
14660        vpxor	%xmm0, %xmm4, %xmm4
14661        # ghash_gfmul_avx
14662        vpclmulqdq	$16, %xmm4, %xmm5, %xmm2
14663        vpclmulqdq	$0x01, %xmm4, %xmm5, %xmm1
14664        vpclmulqdq	$0x00, %xmm4, %xmm5, %xmm0
14665        vpclmulqdq	$0x11, %xmm4, %xmm5, %xmm3
14666        vpxor	%xmm1, %xmm2, %xmm2
14667        vpslldq	$8, %xmm2, %xmm1
14668        vpsrldq	$8, %xmm2, %xmm2
14669        vpxor	%xmm1, %xmm0, %xmm6
14670        vpxor	%xmm2, %xmm3, %xmm4
14671        # ghash_mid
14672        vpsrld	$31, %xmm6, %xmm0
14673        vpsrld	$31, %xmm4, %xmm1
14674        vpslld	$0x01, %xmm6, %xmm6
14675        vpslld	$0x01, %xmm4, %xmm4
14676        vpsrldq	$12, %xmm0, %xmm2
14677        vpslldq	$4, %xmm0, %xmm0
14678        vpslldq	$4, %xmm1, %xmm1
14679        vpor	%xmm2, %xmm4, %xmm4
14680        vpor	%xmm0, %xmm6, %xmm6
14681        vpor	%xmm1, %xmm4, %xmm4
14682        # ghash_red
14683        vmovdqu	L_avx2_aes_gcm_mod2_128(%rip), %xmm2
14684        vpclmulqdq	$16, %xmm2, %xmm6, %xmm0
14685        vpshufd	$0x4e, %xmm6, %xmm1
14686        vpxor	%xmm0, %xmm1, %xmm1
14687        vpclmulqdq	$16, %xmm2, %xmm1, %xmm0
14688        vpshufd	$0x4e, %xmm1, %xmm1
14689        vpxor	%xmm0, %xmm1, %xmm1
14690        vpxor	%xmm1, %xmm4, %xmm4
14691        addl	$16, %ecx
14692        cmpl	%edx, %ecx
14693        jl	L_AES_GCM_init_avx2_calc_iv_16_loop
14694        movl	%r11d, %edx
14695        cmpl	%edx, %ecx
14696        je	L_AES_GCM_init_avx2_calc_iv_done
14697L_AES_GCM_init_avx2_calc_iv_lt16:
14698        vpxor	%xmm0, %xmm0, %xmm0
14699        xorl	%ebx, %ebx
14700        vmovdqu	%xmm0, (%rsp)
14701L_AES_GCM_init_avx2_calc_iv_loop:
14702        movzbl	(%r10,%rcx,1), %r12d
14703        movb	%r12b, (%rsp,%rbx,1)
14704        incl	%ecx
14705        incl	%ebx
14706        cmpl	%edx, %ecx
14707        jl	L_AES_GCM_init_avx2_calc_iv_loop
14708        vmovdqu	(%rsp), %xmm0
14709        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0
14710        vpxor	%xmm0, %xmm4, %xmm4
14711        # ghash_gfmul_avx
14712        vpclmulqdq	$16, %xmm4, %xmm5, %xmm2
14713        vpclmulqdq	$0x01, %xmm4, %xmm5, %xmm1
14714        vpclmulqdq	$0x00, %xmm4, %xmm5, %xmm0
14715        vpclmulqdq	$0x11, %xmm4, %xmm5, %xmm3
14716        vpxor	%xmm1, %xmm2, %xmm2
14717        vpslldq	$8, %xmm2, %xmm1
14718        vpsrldq	$8, %xmm2, %xmm2
14719        vpxor	%xmm1, %xmm0, %xmm6
14720        vpxor	%xmm2, %xmm3, %xmm4
14721        # ghash_mid
14722        vpsrld	$31, %xmm6, %xmm0
14723        vpsrld	$31, %xmm4, %xmm1
14724        vpslld	$0x01, %xmm6, %xmm6
14725        vpslld	$0x01, %xmm4, %xmm4
14726        vpsrldq	$12, %xmm0, %xmm2
14727        vpslldq	$4, %xmm0, %xmm0
14728        vpslldq	$4, %xmm1, %xmm1
14729        vpor	%xmm2, %xmm4, %xmm4
14730        vpor	%xmm0, %xmm6, %xmm6
14731        vpor	%xmm1, %xmm4, %xmm4
14732        # ghash_red
14733        vmovdqu	L_avx2_aes_gcm_mod2_128(%rip), %xmm2
14734        vpclmulqdq	$16, %xmm2, %xmm6, %xmm0
14735        vpshufd	$0x4e, %xmm6, %xmm1
14736        vpxor	%xmm0, %xmm1, %xmm1
14737        vpclmulqdq	$16, %xmm2, %xmm1, %xmm0
14738        vpshufd	$0x4e, %xmm1, %xmm1
14739        vpxor	%xmm0, %xmm1, %xmm1
14740        vpxor	%xmm1, %xmm4, %xmm4
14741L_AES_GCM_init_avx2_calc_iv_done:
14742        # T = Encrypt counter
14743        vpxor	%xmm0, %xmm0, %xmm0
14744        shll	$3, %edx
14745        vmovq	%rdx, %xmm0
14746        vpxor	%xmm0, %xmm4, %xmm4
14747        # ghash_gfmul_avx
14748        vpclmulqdq	$16, %xmm4, %xmm5, %xmm2
14749        vpclmulqdq	$0x01, %xmm4, %xmm5, %xmm1
14750        vpclmulqdq	$0x00, %xmm4, %xmm5, %xmm0
14751        vpclmulqdq	$0x11, %xmm4, %xmm5, %xmm3
14752        vpxor	%xmm1, %xmm2, %xmm2
14753        vpslldq	$8, %xmm2, %xmm1
14754        vpsrldq	$8, %xmm2, %xmm2
14755        vpxor	%xmm1, %xmm0, %xmm6
14756        vpxor	%xmm2, %xmm3, %xmm4
14757        # ghash_mid
14758        vpsrld	$31, %xmm6, %xmm0
14759        vpsrld	$31, %xmm4, %xmm1
14760        vpslld	$0x01, %xmm6, %xmm6
14761        vpslld	$0x01, %xmm4, %xmm4
14762        vpsrldq	$12, %xmm0, %xmm2
14763        vpslldq	$4, %xmm0, %xmm0
14764        vpslldq	$4, %xmm1, %xmm1
14765        vpor	%xmm2, %xmm4, %xmm4
14766        vpor	%xmm0, %xmm6, %xmm6
14767        vpor	%xmm1, %xmm4, %xmm4
14768        # ghash_red
14769        vmovdqu	L_avx2_aes_gcm_mod2_128(%rip), %xmm2
14770        vpclmulqdq	$16, %xmm2, %xmm6, %xmm0
14771        vpshufd	$0x4e, %xmm6, %xmm1
14772        vpxor	%xmm0, %xmm1, %xmm1
14773        vpclmulqdq	$16, %xmm2, %xmm1, %xmm0
14774        vpshufd	$0x4e, %xmm1, %xmm1
14775        vpxor	%xmm0, %xmm1, %xmm1
14776        vpxor	%xmm1, %xmm4, %xmm4
14777        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm4, %xmm4
14778        #   Encrypt counter
14779        vmovdqu	(%rdi), %xmm7
14780        vpxor	%xmm4, %xmm7, %xmm7
14781        vaesenc	16(%rdi), %xmm7, %xmm7
14782        vaesenc	32(%rdi), %xmm7, %xmm7
14783        vaesenc	48(%rdi), %xmm7, %xmm7
14784        vaesenc	64(%rdi), %xmm7, %xmm7
14785        vaesenc	80(%rdi), %xmm7, %xmm7
14786        vaesenc	96(%rdi), %xmm7, %xmm7
14787        vaesenc	112(%rdi), %xmm7, %xmm7
14788        vaesenc	128(%rdi), %xmm7, %xmm7
14789        vaesenc	144(%rdi), %xmm7, %xmm7
14790        cmpl	$11, %esi
14791        vmovdqu	160(%rdi), %xmm0
14792        jl	L_AES_GCM_init_avx2_calc_iv_2_aesenc_avx_last
14793        vaesenc	%xmm0, %xmm7, %xmm7
14794        vaesenc	176(%rdi), %xmm7, %xmm7
14795        cmpl	$13, %esi
14796        vmovdqu	192(%rdi), %xmm0
14797        jl	L_AES_GCM_init_avx2_calc_iv_2_aesenc_avx_last
14798        vaesenc	%xmm0, %xmm7, %xmm7
14799        vaesenc	208(%rdi), %xmm7, %xmm7
14800        vmovdqu	224(%rdi), %xmm0
14801L_AES_GCM_init_avx2_calc_iv_2_aesenc_avx_last:
14802        vaesenclast	%xmm0, %xmm7, %xmm7
14803        jmp	L_AES_GCM_init_avx2_iv_done
14804L_AES_GCM_init_avx2_iv_12:
14805        # # Calculate values when IV is 12 bytes
14806        # Set counter based on IV
14807        vmovdqu	L_avx2_aes_gcm_bswap_one(%rip), %xmm4
14808        vmovdqu	(%rdi), %xmm5
14809        vpblendd	$7, (%r10), %xmm4, %xmm4
14810        # H = Encrypt X(=0) and T = Encrypt counter
14811        vmovdqu	16(%rdi), %xmm6
14812        vpxor	%xmm5, %xmm4, %xmm7
14813        vaesenc	%xmm6, %xmm5, %xmm5
14814        vaesenc	%xmm6, %xmm7, %xmm7
14815        vmovdqu	32(%rdi), %xmm0
14816        vaesenc	%xmm0, %xmm5, %xmm5
14817        vaesenc	%xmm0, %xmm7, %xmm7
14818        vmovdqu	48(%rdi), %xmm0
14819        vaesenc	%xmm0, %xmm5, %xmm5
14820        vaesenc	%xmm0, %xmm7, %xmm7
14821        vmovdqu	64(%rdi), %xmm0
14822        vaesenc	%xmm0, %xmm5, %xmm5
14823        vaesenc	%xmm0, %xmm7, %xmm7
14824        vmovdqu	80(%rdi), %xmm0
14825        vaesenc	%xmm0, %xmm5, %xmm5
14826        vaesenc	%xmm0, %xmm7, %xmm7
14827        vmovdqu	96(%rdi), %xmm0
14828        vaesenc	%xmm0, %xmm5, %xmm5
14829        vaesenc	%xmm0, %xmm7, %xmm7
14830        vmovdqu	112(%rdi), %xmm0
14831        vaesenc	%xmm0, %xmm5, %xmm5
14832        vaesenc	%xmm0, %xmm7, %xmm7
14833        vmovdqu	128(%rdi), %xmm0
14834        vaesenc	%xmm0, %xmm5, %xmm5
14835        vaesenc	%xmm0, %xmm7, %xmm7
14836        vmovdqu	144(%rdi), %xmm0
14837        vaesenc	%xmm0, %xmm5, %xmm5
14838        vaesenc	%xmm0, %xmm7, %xmm7
14839        cmpl	$11, %esi
14840        vmovdqu	160(%rdi), %xmm0
14841        jl	L_AES_GCM_init_avx2_calc_iv_12_last
14842        vaesenc	%xmm0, %xmm5, %xmm5
14843        vaesenc	%xmm0, %xmm7, %xmm7
14844        vmovdqu	176(%rdi), %xmm0
14845        vaesenc	%xmm0, %xmm5, %xmm5
14846        vaesenc	%xmm0, %xmm7, %xmm7
14847        cmpl	$13, %esi
14848        vmovdqu	192(%rdi), %xmm0
14849        jl	L_AES_GCM_init_avx2_calc_iv_12_last
14850        vaesenc	%xmm0, %xmm5, %xmm5
14851        vaesenc	%xmm0, %xmm7, %xmm7
14852        vmovdqu	208(%rdi), %xmm0
14853        vaesenc	%xmm0, %xmm5, %xmm5
14854        vaesenc	%xmm0, %xmm7, %xmm7
14855        vmovdqu	224(%rdi), %xmm0
14856L_AES_GCM_init_avx2_calc_iv_12_last:
14857        vaesenclast	%xmm0, %xmm5, %xmm5
14858        vaesenclast	%xmm0, %xmm7, %xmm7
14859        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm5, %xmm5
14860L_AES_GCM_init_avx2_iv_done:
14861        vmovdqu	%xmm7, (%rax)
14862        vpshufb	L_avx2_aes_gcm_bswap_epi64(%rip), %xmm4, %xmm4
14863        vpaddd	L_avx2_aes_gcm_one(%rip), %xmm4, %xmm4
14864        vmovdqu	%xmm5, (%r8)
14865        vmovdqu	%xmm4, (%r9)
14866        vzeroupper
14867        addq	$16, %rsp
14868        popq	%r12
14869        popq	%rbx
14870        repz retq
14871#ifndef __APPLE__
14872.size	AES_GCM_init_avx2,.-AES_GCM_init_avx2
14873#endif /* __APPLE__ */
14874#ifndef __APPLE__
14875.text
14876.globl	AES_GCM_aad_update_avx2
14877.type	AES_GCM_aad_update_avx2,@function
14878.align	16
14879AES_GCM_aad_update_avx2:
14880#else
14881.section	__TEXT,__text
14882.globl	_AES_GCM_aad_update_avx2
14883.p2align	4
14884_AES_GCM_aad_update_avx2:
14885#endif /* __APPLE__ */
14886        movq	%rcx, %rax
14887        vmovdqu	(%rdx), %xmm4
14888        vmovdqu	(%rax), %xmm5
14889        xorl	%ecx, %ecx
14890L_AES_GCM_aad_update_avx2_16_loop:
14891        vmovdqu	(%rdi,%rcx,1), %xmm0
14892        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0
14893        vpxor	%xmm0, %xmm4, %xmm4
14894        # ghash_gfmul_avx
14895        vpclmulqdq	$16, %xmm4, %xmm5, %xmm2
14896        vpclmulqdq	$0x01, %xmm4, %xmm5, %xmm1
14897        vpclmulqdq	$0x00, %xmm4, %xmm5, %xmm0
14898        vpclmulqdq	$0x11, %xmm4, %xmm5, %xmm3
14899        vpxor	%xmm1, %xmm2, %xmm2
14900        vpslldq	$8, %xmm2, %xmm1
14901        vpsrldq	$8, %xmm2, %xmm2
14902        vpxor	%xmm1, %xmm0, %xmm6
14903        vpxor	%xmm2, %xmm3, %xmm4
14904        # ghash_mid
14905        vpsrld	$31, %xmm6, %xmm0
14906        vpsrld	$31, %xmm4, %xmm1
14907        vpslld	$0x01, %xmm6, %xmm6
14908        vpslld	$0x01, %xmm4, %xmm4
14909        vpsrldq	$12, %xmm0, %xmm2
14910        vpslldq	$4, %xmm0, %xmm0
14911        vpslldq	$4, %xmm1, %xmm1
14912        vpor	%xmm2, %xmm4, %xmm4
14913        vpor	%xmm0, %xmm6, %xmm6
14914        vpor	%xmm1, %xmm4, %xmm4
14915        # ghash_red
14916        vmovdqu	L_avx2_aes_gcm_mod2_128(%rip), %xmm2
14917        vpclmulqdq	$16, %xmm2, %xmm6, %xmm0
14918        vpshufd	$0x4e, %xmm6, %xmm1
14919        vpxor	%xmm0, %xmm1, %xmm1
14920        vpclmulqdq	$16, %xmm2, %xmm1, %xmm0
14921        vpshufd	$0x4e, %xmm1, %xmm1
14922        vpxor	%xmm0, %xmm1, %xmm1
14923        vpxor	%xmm1, %xmm4, %xmm4
14924        addl	$16, %ecx
14925        cmpl	%esi, %ecx
14926        jl	L_AES_GCM_aad_update_avx2_16_loop
14927        vmovdqu	%xmm4, (%rdx)
14928        vzeroupper
14929        repz retq
14930#ifndef __APPLE__
14931.size	AES_GCM_aad_update_avx2,.-AES_GCM_aad_update_avx2
14932#endif /* __APPLE__ */
14933#ifndef __APPLE__
14934.text
14935.globl	AES_GCM_encrypt_block_avx2
14936.type	AES_GCM_encrypt_block_avx2,@function
14937.align	16
14938AES_GCM_encrypt_block_avx2:
14939#else
14940.section	__TEXT,__text
14941.globl	_AES_GCM_encrypt_block_avx2
14942.p2align	4
14943_AES_GCM_encrypt_block_avx2:
14944#endif /* __APPLE__ */
14945        movq	%rdx, %r10
14946        movq	%rcx, %r11
14947        subq	$0x98, %rsp
14948        vmovdqu	(%r8), %xmm3
14949        # aesenc_block
14950        vmovdqu	%xmm3, %xmm1
14951        vpshufb	L_avx2_aes_gcm_bswap_epi64(%rip), %xmm1, %xmm0
14952        vpaddd	L_avx2_aes_gcm_one(%rip), %xmm1, %xmm1
14953        vpxor	(%rdi), %xmm0, %xmm0
14954        vmovdqu	16(%rdi), %xmm2
14955        vaesenc	%xmm2, %xmm0, %xmm0
14956        vmovdqu	32(%rdi), %xmm2
14957        vaesenc	%xmm2, %xmm0, %xmm0
14958        vmovdqu	48(%rdi), %xmm2
14959        vaesenc	%xmm2, %xmm0, %xmm0
14960        vmovdqu	64(%rdi), %xmm2
14961        vaesenc	%xmm2, %xmm0, %xmm0
14962        vmovdqu	80(%rdi), %xmm2
14963        vaesenc	%xmm2, %xmm0, %xmm0
14964        vmovdqu	96(%rdi), %xmm2
14965        vaesenc	%xmm2, %xmm0, %xmm0
14966        vmovdqu	112(%rdi), %xmm2
14967        vaesenc	%xmm2, %xmm0, %xmm0
14968        vmovdqu	128(%rdi), %xmm2
14969        vaesenc	%xmm2, %xmm0, %xmm0
14970        vmovdqu	144(%rdi), %xmm2
14971        vaesenc	%xmm2, %xmm0, %xmm0
14972        vmovdqu	%xmm1, %xmm3
14973        cmpl	$11, %esi
14974        vmovdqu	160(%rdi), %xmm1
14975        jl	L_AES_GCM_encrypt_block_avx2_aesenc_block_last
14976        vaesenc	%xmm1, %xmm0, %xmm0
14977        vmovdqu	176(%rdi), %xmm2
14978        vaesenc	%xmm2, %xmm0, %xmm0
14979        cmpl	$13, %esi
14980        vmovdqu	192(%rdi), %xmm1
14981        jl	L_AES_GCM_encrypt_block_avx2_aesenc_block_last
14982        vaesenc	%xmm1, %xmm0, %xmm0
14983        vmovdqu	208(%rdi), %xmm2
14984        vaesenc	%xmm2, %xmm0, %xmm0
14985        vmovdqu	224(%rdi), %xmm1
14986L_AES_GCM_encrypt_block_avx2_aesenc_block_last:
14987        vaesenclast	%xmm1, %xmm0, %xmm0
14988        vmovdqu	(%r11), %xmm1
14989        vpxor	%xmm1, %xmm0, %xmm0
14990        vmovdqu	%xmm0, (%r10)
14991        vmovdqu	%xmm3, (%r8)
14992        vzeroupper
14993        addq	$0x98, %rsp
14994        repz retq
14995#ifndef __APPLE__
14996.size	AES_GCM_encrypt_block_avx2,.-AES_GCM_encrypt_block_avx2
14997#endif /* __APPLE__ */
14998#ifndef __APPLE__
14999.text
15000.globl	AES_GCM_ghash_block_avx2
15001.type	AES_GCM_ghash_block_avx2,@function
15002.align	16
15003AES_GCM_ghash_block_avx2:
15004#else
15005.section	__TEXT,__text
15006.globl	_AES_GCM_ghash_block_avx2
15007.p2align	4
15008_AES_GCM_ghash_block_avx2:
15009#endif /* __APPLE__ */
15010        vmovdqu	(%rsi), %xmm4
15011        vmovdqu	(%rdx), %xmm5
15012        vmovdqu	(%rdi), %xmm0
15013        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0
15014        vpxor	%xmm0, %xmm4, %xmm4
15015        # ghash_gfmul_avx
15016        vpclmulqdq	$16, %xmm4, %xmm5, %xmm2
15017        vpclmulqdq	$0x01, %xmm4, %xmm5, %xmm1
15018        vpclmulqdq	$0x00, %xmm4, %xmm5, %xmm0
15019        vpclmulqdq	$0x11, %xmm4, %xmm5, %xmm3
15020        vpxor	%xmm1, %xmm2, %xmm2
15021        vpslldq	$8, %xmm2, %xmm1
15022        vpsrldq	$8, %xmm2, %xmm2
15023        vpxor	%xmm1, %xmm0, %xmm6
15024        vpxor	%xmm2, %xmm3, %xmm4
15025        # ghash_mid
15026        vpsrld	$31, %xmm6, %xmm0
15027        vpsrld	$31, %xmm4, %xmm1
15028        vpslld	$0x01, %xmm6, %xmm6
15029        vpslld	$0x01, %xmm4, %xmm4
15030        vpsrldq	$12, %xmm0, %xmm2
15031        vpslldq	$4, %xmm0, %xmm0
15032        vpslldq	$4, %xmm1, %xmm1
15033        vpor	%xmm2, %xmm4, %xmm4
15034        vpor	%xmm0, %xmm6, %xmm6
15035        vpor	%xmm1, %xmm4, %xmm4
15036        # ghash_red
15037        vmovdqu	L_avx2_aes_gcm_mod2_128(%rip), %xmm2
15038        vpclmulqdq	$16, %xmm2, %xmm6, %xmm0
15039        vpshufd	$0x4e, %xmm6, %xmm1
15040        vpxor	%xmm0, %xmm1, %xmm1
15041        vpclmulqdq	$16, %xmm2, %xmm1, %xmm0
15042        vpshufd	$0x4e, %xmm1, %xmm1
15043        vpxor	%xmm0, %xmm1, %xmm1
15044        vpxor	%xmm1, %xmm4, %xmm4
15045        vmovdqu	%xmm4, (%rsi)
15046        vzeroupper
15047        repz retq
15048#ifndef __APPLE__
15049.size	AES_GCM_ghash_block_avx2,.-AES_GCM_ghash_block_avx2
15050#endif /* __APPLE__ */
15051#ifndef __APPLE__
15052.text
15053.globl	AES_GCM_encrypt_update_avx2
15054.type	AES_GCM_encrypt_update_avx2,@function
15055.align	16
15056AES_GCM_encrypt_update_avx2:
15057#else
15058.section	__TEXT,__text
15059.globl	_AES_GCM_encrypt_update_avx2
15060.p2align	4
15061_AES_GCM_encrypt_update_avx2:
15062#endif /* __APPLE__ */
15063        pushq	%r12
15064        pushq	%r13
15065        pushq	%r14
15066        movq	%rdx, %r10
15067        movq	%rcx, %r11
15068        movq	32(%rsp), %rax
15069        movq	40(%rsp), %r12
15070        subq	$0x98, %rsp
15071        vmovdqu	(%r9), %xmm6
15072        vmovdqu	(%rax), %xmm5
15073        vmovdqu	(%r12), %xmm4
15074        vpsrlq	$63, %xmm5, %xmm1
15075        vpsllq	$0x01, %xmm5, %xmm0
15076        vpslldq	$8, %xmm1, %xmm1
15077        vpor	%xmm1, %xmm0, %xmm0
15078        vpshufd	$0xff, %xmm5, %xmm5
15079        vpsrad	$31, %xmm5, %xmm5
15080        vpand	L_avx2_aes_gcm_mod2_128(%rip), %xmm5, %xmm5
15081        vpxor	%xmm0, %xmm5, %xmm5
15082        xorl	%r14d, %r14d
15083        cmpl	$0x80, %r8d
15084        movl	%r8d, %r13d
15085        jl	L_AES_GCM_encrypt_update_avx2_done_128
15086        andl	$0xffffff80, %r13d
15087        vmovdqu	%xmm4, 128(%rsp)
15088        vmovdqu	L_avx2_aes_gcm_mod2_128(%rip), %xmm3
15089        # H ^ 1 and H ^ 2
15090        vpclmulqdq	$0x00, %xmm5, %xmm5, %xmm9
15091        vpclmulqdq	$0x11, %xmm5, %xmm5, %xmm10
15092        vpclmulqdq	$16, %xmm3, %xmm9, %xmm8
15093        vpshufd	$0x4e, %xmm9, %xmm9
15094        vpxor	%xmm8, %xmm9, %xmm9
15095        vpclmulqdq	$16, %xmm3, %xmm9, %xmm8
15096        vpshufd	$0x4e, %xmm9, %xmm9
15097        vpxor	%xmm8, %xmm9, %xmm9
15098        vpxor	%xmm9, %xmm10, %xmm0
15099        vmovdqu	%xmm5, (%rsp)
15100        vmovdqu	%xmm0, 16(%rsp)
15101        # H ^ 3 and H ^ 4
15102        vpclmulqdq	$16, %xmm5, %xmm0, %xmm11
15103        vpclmulqdq	$0x01, %xmm5, %xmm0, %xmm10
15104        vpclmulqdq	$0x00, %xmm5, %xmm0, %xmm9
15105        vpclmulqdq	$0x11, %xmm5, %xmm0, %xmm12
15106        vpclmulqdq	$0x00, %xmm0, %xmm0, %xmm13
15107        vpclmulqdq	$0x11, %xmm0, %xmm0, %xmm14
15108        vpxor	%xmm10, %xmm11, %xmm11
15109        vpslldq	$8, %xmm11, %xmm10
15110        vpsrldq	$8, %xmm11, %xmm11
15111        vpxor	%xmm9, %xmm10, %xmm10
15112        vpclmulqdq	$16, %xmm3, %xmm13, %xmm8
15113        vpclmulqdq	$16, %xmm3, %xmm10, %xmm9
15114        vpshufd	$0x4e, %xmm10, %xmm10
15115        vpshufd	$0x4e, %xmm13, %xmm13
15116        vpxor	%xmm9, %xmm10, %xmm10
15117        vpxor	%xmm8, %xmm13, %xmm13
15118        vpclmulqdq	$16, %xmm3, %xmm10, %xmm9
15119        vpclmulqdq	$16, %xmm3, %xmm13, %xmm8
15120        vpshufd	$0x4e, %xmm10, %xmm10
15121        vpshufd	$0x4e, %xmm13, %xmm13
15122        vpxor	%xmm11, %xmm12, %xmm12
15123        vpxor	%xmm8, %xmm13, %xmm13
15124        vpxor	%xmm12, %xmm10, %xmm10
15125        vpxor	%xmm14, %xmm13, %xmm2
15126        vpxor	%xmm9, %xmm10, %xmm1
15127        vmovdqu	%xmm1, 32(%rsp)
15128        vmovdqu	%xmm2, 48(%rsp)
15129        # H ^ 5 and H ^ 6
15130        vpclmulqdq	$16, %xmm0, %xmm1, %xmm11
15131        vpclmulqdq	$0x01, %xmm0, %xmm1, %xmm10
15132        vpclmulqdq	$0x00, %xmm0, %xmm1, %xmm9
15133        vpclmulqdq	$0x11, %xmm0, %xmm1, %xmm12
15134        vpclmulqdq	$0x00, %xmm1, %xmm1, %xmm13
15135        vpclmulqdq	$0x11, %xmm1, %xmm1, %xmm14
15136        vpxor	%xmm10, %xmm11, %xmm11
15137        vpslldq	$8, %xmm11, %xmm10
15138        vpsrldq	$8, %xmm11, %xmm11
15139        vpxor	%xmm9, %xmm10, %xmm10
15140        vpclmulqdq	$16, %xmm3, %xmm13, %xmm8
15141        vpclmulqdq	$16, %xmm3, %xmm10, %xmm9
15142        vpshufd	$0x4e, %xmm10, %xmm10
15143        vpshufd	$0x4e, %xmm13, %xmm13
15144        vpxor	%xmm9, %xmm10, %xmm10
15145        vpxor	%xmm8, %xmm13, %xmm13
15146        vpclmulqdq	$16, %xmm3, %xmm10, %xmm9
15147        vpclmulqdq	$16, %xmm3, %xmm13, %xmm8
15148        vpshufd	$0x4e, %xmm10, %xmm10
15149        vpshufd	$0x4e, %xmm13, %xmm13
15150        vpxor	%xmm11, %xmm12, %xmm12
15151        vpxor	%xmm8, %xmm13, %xmm13
15152        vpxor	%xmm12, %xmm10, %xmm10
15153        vpxor	%xmm14, %xmm13, %xmm0
15154        vpxor	%xmm9, %xmm10, %xmm7
15155        vmovdqu	%xmm7, 64(%rsp)
15156        vmovdqu	%xmm0, 80(%rsp)
15157        # H ^ 7 and H ^ 8
15158        vpclmulqdq	$16, %xmm1, %xmm2, %xmm11
15159        vpclmulqdq	$0x01, %xmm1, %xmm2, %xmm10
15160        vpclmulqdq	$0x00, %xmm1, %xmm2, %xmm9
15161        vpclmulqdq	$0x11, %xmm1, %xmm2, %xmm12
15162        vpclmulqdq	$0x00, %xmm2, %xmm2, %xmm13
15163        vpclmulqdq	$0x11, %xmm2, %xmm2, %xmm14
15164        vpxor	%xmm10, %xmm11, %xmm11
15165        vpslldq	$8, %xmm11, %xmm10
15166        vpsrldq	$8, %xmm11, %xmm11
15167        vpxor	%xmm9, %xmm10, %xmm10
15168        vpclmulqdq	$16, %xmm3, %xmm13, %xmm8
15169        vpclmulqdq	$16, %xmm3, %xmm10, %xmm9
15170        vpshufd	$0x4e, %xmm10, %xmm10
15171        vpshufd	$0x4e, %xmm13, %xmm13
15172        vpxor	%xmm9, %xmm10, %xmm10
15173        vpxor	%xmm8, %xmm13, %xmm13
15174        vpclmulqdq	$16, %xmm3, %xmm10, %xmm9
15175        vpclmulqdq	$16, %xmm3, %xmm13, %xmm8
15176        vpshufd	$0x4e, %xmm10, %xmm10
15177        vpshufd	$0x4e, %xmm13, %xmm13
15178        vpxor	%xmm11, %xmm12, %xmm12
15179        vpxor	%xmm8, %xmm13, %xmm13
15180        vpxor	%xmm12, %xmm10, %xmm10
15181        vpxor	%xmm14, %xmm13, %xmm0
15182        vpxor	%xmm9, %xmm10, %xmm7
15183        vmovdqu	%xmm7, 96(%rsp)
15184        vmovdqu	%xmm0, 112(%rsp)
15185        # First 128 bytes of input
15186        # aesenc_128
15187        # aesenc_ctr
15188        vmovdqu	128(%rsp), %xmm0
15189        vmovdqu	L_avx2_aes_gcm_bswap_epi64(%rip), %xmm1
15190        vpaddd	L_avx2_aes_gcm_one(%rip), %xmm0, %xmm9
15191        vpshufb	%xmm1, %xmm0, %xmm8
15192        vpaddd	L_avx2_aes_gcm_two(%rip), %xmm0, %xmm10
15193        vpshufb	%xmm1, %xmm9, %xmm9
15194        vpaddd	L_avx2_aes_gcm_three(%rip), %xmm0, %xmm11
15195        vpshufb	%xmm1, %xmm10, %xmm10
15196        vpaddd	L_avx2_aes_gcm_four(%rip), %xmm0, %xmm12
15197        vpshufb	%xmm1, %xmm11, %xmm11
15198        vpaddd	L_avx2_aes_gcm_five(%rip), %xmm0, %xmm13
15199        vpshufb	%xmm1, %xmm12, %xmm12
15200        vpaddd	L_avx2_aes_gcm_six(%rip), %xmm0, %xmm14
15201        vpshufb	%xmm1, %xmm13, %xmm13
15202        vpaddd	L_avx2_aes_gcm_seven(%rip), %xmm0, %xmm15
15203        vpshufb	%xmm1, %xmm14, %xmm14
15204        vpaddd	L_avx2_aes_gcm_eight(%rip), %xmm0, %xmm0
15205        vpshufb	%xmm1, %xmm15, %xmm15
15206        # aesenc_xor
15207        vmovdqu	(%rdi), %xmm7
15208        vmovdqu	%xmm0, 128(%rsp)
15209        vpxor	%xmm7, %xmm8, %xmm8
15210        vpxor	%xmm7, %xmm9, %xmm9
15211        vpxor	%xmm7, %xmm10, %xmm10
15212        vpxor	%xmm7, %xmm11, %xmm11
15213        vpxor	%xmm7, %xmm12, %xmm12
15214        vpxor	%xmm7, %xmm13, %xmm13
15215        vpxor	%xmm7, %xmm14, %xmm14
15216        vpxor	%xmm7, %xmm15, %xmm15
15217        vmovdqu	16(%rdi), %xmm7
15218        vaesenc	%xmm7, %xmm8, %xmm8
15219        vaesenc	%xmm7, %xmm9, %xmm9
15220        vaesenc	%xmm7, %xmm10, %xmm10
15221        vaesenc	%xmm7, %xmm11, %xmm11
15222        vaesenc	%xmm7, %xmm12, %xmm12
15223        vaesenc	%xmm7, %xmm13, %xmm13
15224        vaesenc	%xmm7, %xmm14, %xmm14
15225        vaesenc	%xmm7, %xmm15, %xmm15
15226        vmovdqu	32(%rdi), %xmm7
15227        vaesenc	%xmm7, %xmm8, %xmm8
15228        vaesenc	%xmm7, %xmm9, %xmm9
15229        vaesenc	%xmm7, %xmm10, %xmm10
15230        vaesenc	%xmm7, %xmm11, %xmm11
15231        vaesenc	%xmm7, %xmm12, %xmm12
15232        vaesenc	%xmm7, %xmm13, %xmm13
15233        vaesenc	%xmm7, %xmm14, %xmm14
15234        vaesenc	%xmm7, %xmm15, %xmm15
15235        vmovdqu	48(%rdi), %xmm7
15236        vaesenc	%xmm7, %xmm8, %xmm8
15237        vaesenc	%xmm7, %xmm9, %xmm9
15238        vaesenc	%xmm7, %xmm10, %xmm10
15239        vaesenc	%xmm7, %xmm11, %xmm11
15240        vaesenc	%xmm7, %xmm12, %xmm12
15241        vaesenc	%xmm7, %xmm13, %xmm13
15242        vaesenc	%xmm7, %xmm14, %xmm14
15243        vaesenc	%xmm7, %xmm15, %xmm15
15244        vmovdqu	64(%rdi), %xmm7
15245        vaesenc	%xmm7, %xmm8, %xmm8
15246        vaesenc	%xmm7, %xmm9, %xmm9
15247        vaesenc	%xmm7, %xmm10, %xmm10
15248        vaesenc	%xmm7, %xmm11, %xmm11
15249        vaesenc	%xmm7, %xmm12, %xmm12
15250        vaesenc	%xmm7, %xmm13, %xmm13
15251        vaesenc	%xmm7, %xmm14, %xmm14
15252        vaesenc	%xmm7, %xmm15, %xmm15
15253        vmovdqu	80(%rdi), %xmm7
15254        vaesenc	%xmm7, %xmm8, %xmm8
15255        vaesenc	%xmm7, %xmm9, %xmm9
15256        vaesenc	%xmm7, %xmm10, %xmm10
15257        vaesenc	%xmm7, %xmm11, %xmm11
15258        vaesenc	%xmm7, %xmm12, %xmm12
15259        vaesenc	%xmm7, %xmm13, %xmm13
15260        vaesenc	%xmm7, %xmm14, %xmm14
15261        vaesenc	%xmm7, %xmm15, %xmm15
15262        vmovdqu	96(%rdi), %xmm7
15263        vaesenc	%xmm7, %xmm8, %xmm8
15264        vaesenc	%xmm7, %xmm9, %xmm9
15265        vaesenc	%xmm7, %xmm10, %xmm10
15266        vaesenc	%xmm7, %xmm11, %xmm11
15267        vaesenc	%xmm7, %xmm12, %xmm12
15268        vaesenc	%xmm7, %xmm13, %xmm13
15269        vaesenc	%xmm7, %xmm14, %xmm14
15270        vaesenc	%xmm7, %xmm15, %xmm15
15271        vmovdqu	112(%rdi), %xmm7
15272        vaesenc	%xmm7, %xmm8, %xmm8
15273        vaesenc	%xmm7, %xmm9, %xmm9
15274        vaesenc	%xmm7, %xmm10, %xmm10
15275        vaesenc	%xmm7, %xmm11, %xmm11
15276        vaesenc	%xmm7, %xmm12, %xmm12
15277        vaesenc	%xmm7, %xmm13, %xmm13
15278        vaesenc	%xmm7, %xmm14, %xmm14
15279        vaesenc	%xmm7, %xmm15, %xmm15
15280        vmovdqu	128(%rdi), %xmm7
15281        vaesenc	%xmm7, %xmm8, %xmm8
15282        vaesenc	%xmm7, %xmm9, %xmm9
15283        vaesenc	%xmm7, %xmm10, %xmm10
15284        vaesenc	%xmm7, %xmm11, %xmm11
15285        vaesenc	%xmm7, %xmm12, %xmm12
15286        vaesenc	%xmm7, %xmm13, %xmm13
15287        vaesenc	%xmm7, %xmm14, %xmm14
15288        vaesenc	%xmm7, %xmm15, %xmm15
15289        vmovdqu	144(%rdi), %xmm7
15290        vaesenc	%xmm7, %xmm8, %xmm8
15291        vaesenc	%xmm7, %xmm9, %xmm9
15292        vaesenc	%xmm7, %xmm10, %xmm10
15293        vaesenc	%xmm7, %xmm11, %xmm11
15294        vaesenc	%xmm7, %xmm12, %xmm12
15295        vaesenc	%xmm7, %xmm13, %xmm13
15296        vaesenc	%xmm7, %xmm14, %xmm14
15297        vaesenc	%xmm7, %xmm15, %xmm15
15298        cmpl	$11, %esi
15299        vmovdqu	160(%rdi), %xmm7
15300        jl	L_AES_GCM_encrypt_update_avx2_aesenc_128_enc_done
15301        vaesenc	%xmm7, %xmm8, %xmm8
15302        vaesenc	%xmm7, %xmm9, %xmm9
15303        vaesenc	%xmm7, %xmm10, %xmm10
15304        vaesenc	%xmm7, %xmm11, %xmm11
15305        vaesenc	%xmm7, %xmm12, %xmm12
15306        vaesenc	%xmm7, %xmm13, %xmm13
15307        vaesenc	%xmm7, %xmm14, %xmm14
15308        vaesenc	%xmm7, %xmm15, %xmm15
15309        vmovdqu	176(%rdi), %xmm7
15310        vaesenc	%xmm7, %xmm8, %xmm8
15311        vaesenc	%xmm7, %xmm9, %xmm9
15312        vaesenc	%xmm7, %xmm10, %xmm10
15313        vaesenc	%xmm7, %xmm11, %xmm11
15314        vaesenc	%xmm7, %xmm12, %xmm12
15315        vaesenc	%xmm7, %xmm13, %xmm13
15316        vaesenc	%xmm7, %xmm14, %xmm14
15317        vaesenc	%xmm7, %xmm15, %xmm15
15318        cmpl	$13, %esi
15319        vmovdqu	192(%rdi), %xmm7
15320        jl	L_AES_GCM_encrypt_update_avx2_aesenc_128_enc_done
15321        vaesenc	%xmm7, %xmm8, %xmm8
15322        vaesenc	%xmm7, %xmm9, %xmm9
15323        vaesenc	%xmm7, %xmm10, %xmm10
15324        vaesenc	%xmm7, %xmm11, %xmm11
15325        vaesenc	%xmm7, %xmm12, %xmm12
15326        vaesenc	%xmm7, %xmm13, %xmm13
15327        vaesenc	%xmm7, %xmm14, %xmm14
15328        vaesenc	%xmm7, %xmm15, %xmm15
15329        vmovdqu	208(%rdi), %xmm7
15330        vaesenc	%xmm7, %xmm8, %xmm8
15331        vaesenc	%xmm7, %xmm9, %xmm9
15332        vaesenc	%xmm7, %xmm10, %xmm10
15333        vaesenc	%xmm7, %xmm11, %xmm11
15334        vaesenc	%xmm7, %xmm12, %xmm12
15335        vaesenc	%xmm7, %xmm13, %xmm13
15336        vaesenc	%xmm7, %xmm14, %xmm14
15337        vaesenc	%xmm7, %xmm15, %xmm15
15338        vmovdqu	224(%rdi), %xmm7
15339L_AES_GCM_encrypt_update_avx2_aesenc_128_enc_done:
15340        # aesenc_last
15341        vaesenclast	%xmm7, %xmm8, %xmm8
15342        vaesenclast	%xmm7, %xmm9, %xmm9
15343        vaesenclast	%xmm7, %xmm10, %xmm10
15344        vaesenclast	%xmm7, %xmm11, %xmm11
15345        vmovdqu	(%r11), %xmm0
15346        vmovdqu	16(%r11), %xmm1
15347        vmovdqu	32(%r11), %xmm2
15348        vmovdqu	48(%r11), %xmm3
15349        vpxor	%xmm0, %xmm8, %xmm8
15350        vpxor	%xmm1, %xmm9, %xmm9
15351        vpxor	%xmm2, %xmm10, %xmm10
15352        vpxor	%xmm3, %xmm11, %xmm11
15353        vmovdqu	%xmm8, (%r10)
15354        vmovdqu	%xmm9, 16(%r10)
15355        vmovdqu	%xmm10, 32(%r10)
15356        vmovdqu	%xmm11, 48(%r10)
15357        vaesenclast	%xmm7, %xmm12, %xmm12
15358        vaesenclast	%xmm7, %xmm13, %xmm13
15359        vaesenclast	%xmm7, %xmm14, %xmm14
15360        vaesenclast	%xmm7, %xmm15, %xmm15
15361        vmovdqu	64(%r11), %xmm0
15362        vmovdqu	80(%r11), %xmm1
15363        vmovdqu	96(%r11), %xmm2
15364        vmovdqu	112(%r11), %xmm3
15365        vpxor	%xmm0, %xmm12, %xmm12
15366        vpxor	%xmm1, %xmm13, %xmm13
15367        vpxor	%xmm2, %xmm14, %xmm14
15368        vpxor	%xmm3, %xmm15, %xmm15
15369        vmovdqu	%xmm12, 64(%r10)
15370        vmovdqu	%xmm13, 80(%r10)
15371        vmovdqu	%xmm14, 96(%r10)
15372        vmovdqu	%xmm15, 112(%r10)
15373        cmpl	$0x80, %r13d
15374        movl	$0x80, %r14d
15375        jle	L_AES_GCM_encrypt_update_avx2_end_128
15376        # More 128 bytes of input
15377L_AES_GCM_encrypt_update_avx2_ghash_128:
15378        # aesenc_128_ghash
15379        leaq	(%r11,%r14,1), %rcx
15380        leaq	(%r10,%r14,1), %rdx
15381        # aesenc_ctr
15382        vmovdqu	128(%rsp), %xmm0
15383        vmovdqu	L_avx2_aes_gcm_bswap_epi64(%rip), %xmm1
15384        vpaddd	L_avx2_aes_gcm_one(%rip), %xmm0, %xmm9
15385        vpshufb	%xmm1, %xmm0, %xmm8
15386        vpaddd	L_avx2_aes_gcm_two(%rip), %xmm0, %xmm10
15387        vpshufb	%xmm1, %xmm9, %xmm9
15388        vpaddd	L_avx2_aes_gcm_three(%rip), %xmm0, %xmm11
15389        vpshufb	%xmm1, %xmm10, %xmm10
15390        vpaddd	L_avx2_aes_gcm_four(%rip), %xmm0, %xmm12
15391        vpshufb	%xmm1, %xmm11, %xmm11
15392        vpaddd	L_avx2_aes_gcm_five(%rip), %xmm0, %xmm13
15393        vpshufb	%xmm1, %xmm12, %xmm12
15394        vpaddd	L_avx2_aes_gcm_six(%rip), %xmm0, %xmm14
15395        vpshufb	%xmm1, %xmm13, %xmm13
15396        vpaddd	L_avx2_aes_gcm_seven(%rip), %xmm0, %xmm15
15397        vpshufb	%xmm1, %xmm14, %xmm14
15398        vpaddd	L_avx2_aes_gcm_eight(%rip), %xmm0, %xmm0
15399        vpshufb	%xmm1, %xmm15, %xmm15
15400        # aesenc_xor
15401        vmovdqu	(%rdi), %xmm7
15402        vmovdqu	%xmm0, 128(%rsp)
15403        vpxor	%xmm7, %xmm8, %xmm8
15404        vpxor	%xmm7, %xmm9, %xmm9
15405        vpxor	%xmm7, %xmm10, %xmm10
15406        vpxor	%xmm7, %xmm11, %xmm11
15407        vpxor	%xmm7, %xmm12, %xmm12
15408        vpxor	%xmm7, %xmm13, %xmm13
15409        vpxor	%xmm7, %xmm14, %xmm14
15410        vpxor	%xmm7, %xmm15, %xmm15
15411        # aesenc_pclmul_1
15412        vmovdqu	-128(%rdx), %xmm1
15413        vmovdqu	16(%rdi), %xmm0
15414        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm1, %xmm1
15415        vmovdqu	112(%rsp), %xmm2
15416        vpxor	%xmm6, %xmm1, %xmm1
15417        vpclmulqdq	$16, %xmm2, %xmm1, %xmm5
15418        vpclmulqdq	$0x01, %xmm2, %xmm1, %xmm3
15419        vpclmulqdq	$0x00, %xmm2, %xmm1, %xmm6
15420        vpclmulqdq	$0x11, %xmm2, %xmm1, %xmm7
15421        vaesenc	%xmm0, %xmm8, %xmm8
15422        vaesenc	%xmm0, %xmm9, %xmm9
15423        vaesenc	%xmm0, %xmm10, %xmm10
15424        vaesenc	%xmm0, %xmm11, %xmm11
15425        vaesenc	%xmm0, %xmm12, %xmm12
15426        vaesenc	%xmm0, %xmm13, %xmm13
15427        vaesenc	%xmm0, %xmm14, %xmm14
15428        vaesenc	%xmm0, %xmm15, %xmm15
15429        # aesenc_pclmul_2
15430        vmovdqu	-112(%rdx), %xmm1
15431        vmovdqu	96(%rsp), %xmm0
15432        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm1, %xmm1
15433        vpxor	%xmm3, %xmm5, %xmm5
15434        vpclmulqdq	$16, %xmm0, %xmm1, %xmm2
15435        vpclmulqdq	$0x01, %xmm0, %xmm1, %xmm3
15436        vpclmulqdq	$0x00, %xmm0, %xmm1, %xmm4
15437        vpclmulqdq	$0x11, %xmm0, %xmm1, %xmm1
15438        vmovdqu	32(%rdi), %xmm0
15439        vpxor	%xmm1, %xmm7, %xmm7
15440        vaesenc	%xmm0, %xmm8, %xmm8
15441        vaesenc	%xmm0, %xmm9, %xmm9
15442        vaesenc	%xmm0, %xmm10, %xmm10
15443        vaesenc	%xmm0, %xmm11, %xmm11
15444        vaesenc	%xmm0, %xmm12, %xmm12
15445        vaesenc	%xmm0, %xmm13, %xmm13
15446        vaesenc	%xmm0, %xmm14, %xmm14
15447        vaesenc	%xmm0, %xmm15, %xmm15
15448        # aesenc_pclmul_n
15449        vmovdqu	-96(%rdx), %xmm1
15450        vmovdqu	80(%rsp), %xmm0
15451        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm1, %xmm1
15452        vpxor	%xmm2, %xmm5, %xmm5
15453        vpclmulqdq	$16, %xmm0, %xmm1, %xmm2
15454        vpxor	%xmm3, %xmm5, %xmm5
15455        vpclmulqdq	$0x01, %xmm0, %xmm1, %xmm3
15456        vpxor	%xmm4, %xmm6, %xmm6
15457        vpclmulqdq	$0x00, %xmm0, %xmm1, %xmm4
15458        vpclmulqdq	$0x11, %xmm0, %xmm1, %xmm1
15459        vmovdqu	48(%rdi), %xmm0
15460        vpxor	%xmm1, %xmm7, %xmm7
15461        vaesenc	%xmm0, %xmm8, %xmm8
15462        vaesenc	%xmm0, %xmm9, %xmm9
15463        vaesenc	%xmm0, %xmm10, %xmm10
15464        vaesenc	%xmm0, %xmm11, %xmm11
15465        vaesenc	%xmm0, %xmm12, %xmm12
15466        vaesenc	%xmm0, %xmm13, %xmm13
15467        vaesenc	%xmm0, %xmm14, %xmm14
15468        vaesenc	%xmm0, %xmm15, %xmm15
15469        # aesenc_pclmul_n
15470        vmovdqu	-80(%rdx), %xmm1
15471        vmovdqu	64(%rsp), %xmm0
15472        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm1, %xmm1
15473        vpxor	%xmm2, %xmm5, %xmm5
15474        vpclmulqdq	$16, %xmm0, %xmm1, %xmm2
15475        vpxor	%xmm3, %xmm5, %xmm5
15476        vpclmulqdq	$0x01, %xmm0, %xmm1, %xmm3
15477        vpxor	%xmm4, %xmm6, %xmm6
15478        vpclmulqdq	$0x00, %xmm0, %xmm1, %xmm4
15479        vpclmulqdq	$0x11, %xmm0, %xmm1, %xmm1
15480        vmovdqu	64(%rdi), %xmm0
15481        vpxor	%xmm1, %xmm7, %xmm7
15482        vaesenc	%xmm0, %xmm8, %xmm8
15483        vaesenc	%xmm0, %xmm9, %xmm9
15484        vaesenc	%xmm0, %xmm10, %xmm10
15485        vaesenc	%xmm0, %xmm11, %xmm11
15486        vaesenc	%xmm0, %xmm12, %xmm12
15487        vaesenc	%xmm0, %xmm13, %xmm13
15488        vaesenc	%xmm0, %xmm14, %xmm14
15489        vaesenc	%xmm0, %xmm15, %xmm15
15490        # aesenc_pclmul_n
15491        vmovdqu	-64(%rdx), %xmm1
15492        vmovdqu	48(%rsp), %xmm0
15493        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm1, %xmm1
15494        vpxor	%xmm2, %xmm5, %xmm5
15495        vpclmulqdq	$16, %xmm0, %xmm1, %xmm2
15496        vpxor	%xmm3, %xmm5, %xmm5
15497        vpclmulqdq	$0x01, %xmm0, %xmm1, %xmm3
15498        vpxor	%xmm4, %xmm6, %xmm6
15499        vpclmulqdq	$0x00, %xmm0, %xmm1, %xmm4
15500        vpclmulqdq	$0x11, %xmm0, %xmm1, %xmm1
15501        vmovdqu	80(%rdi), %xmm0
15502        vpxor	%xmm1, %xmm7, %xmm7
15503        vaesenc	%xmm0, %xmm8, %xmm8
15504        vaesenc	%xmm0, %xmm9, %xmm9
15505        vaesenc	%xmm0, %xmm10, %xmm10
15506        vaesenc	%xmm0, %xmm11, %xmm11
15507        vaesenc	%xmm0, %xmm12, %xmm12
15508        vaesenc	%xmm0, %xmm13, %xmm13
15509        vaesenc	%xmm0, %xmm14, %xmm14
15510        vaesenc	%xmm0, %xmm15, %xmm15
15511        # aesenc_pclmul_n
15512        vmovdqu	-48(%rdx), %xmm1
15513        vmovdqu	32(%rsp), %xmm0
15514        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm1, %xmm1
15515        vpxor	%xmm2, %xmm5, %xmm5
15516        vpclmulqdq	$16, %xmm0, %xmm1, %xmm2
15517        vpxor	%xmm3, %xmm5, %xmm5
15518        vpclmulqdq	$0x01, %xmm0, %xmm1, %xmm3
15519        vpxor	%xmm4, %xmm6, %xmm6
15520        vpclmulqdq	$0x00, %xmm0, %xmm1, %xmm4
15521        vpclmulqdq	$0x11, %xmm0, %xmm1, %xmm1
15522        vmovdqu	96(%rdi), %xmm0
15523        vpxor	%xmm1, %xmm7, %xmm7
15524        vaesenc	%xmm0, %xmm8, %xmm8
15525        vaesenc	%xmm0, %xmm9, %xmm9
15526        vaesenc	%xmm0, %xmm10, %xmm10
15527        vaesenc	%xmm0, %xmm11, %xmm11
15528        vaesenc	%xmm0, %xmm12, %xmm12
15529        vaesenc	%xmm0, %xmm13, %xmm13
15530        vaesenc	%xmm0, %xmm14, %xmm14
15531        vaesenc	%xmm0, %xmm15, %xmm15
15532        # aesenc_pclmul_n
15533        vmovdqu	-32(%rdx), %xmm1
15534        vmovdqu	16(%rsp), %xmm0
15535        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm1, %xmm1
15536        vpxor	%xmm2, %xmm5, %xmm5
15537        vpclmulqdq	$16, %xmm0, %xmm1, %xmm2
15538        vpxor	%xmm3, %xmm5, %xmm5
15539        vpclmulqdq	$0x01, %xmm0, %xmm1, %xmm3
15540        vpxor	%xmm4, %xmm6, %xmm6
15541        vpclmulqdq	$0x00, %xmm0, %xmm1, %xmm4
15542        vpclmulqdq	$0x11, %xmm0, %xmm1, %xmm1
15543        vmovdqu	112(%rdi), %xmm0
15544        vpxor	%xmm1, %xmm7, %xmm7
15545        vaesenc	%xmm0, %xmm8, %xmm8
15546        vaesenc	%xmm0, %xmm9, %xmm9
15547        vaesenc	%xmm0, %xmm10, %xmm10
15548        vaesenc	%xmm0, %xmm11, %xmm11
15549        vaesenc	%xmm0, %xmm12, %xmm12
15550        vaesenc	%xmm0, %xmm13, %xmm13
15551        vaesenc	%xmm0, %xmm14, %xmm14
15552        vaesenc	%xmm0, %xmm15, %xmm15
15553        # aesenc_pclmul_n
15554        vmovdqu	-16(%rdx), %xmm1
15555        vmovdqu	(%rsp), %xmm0
15556        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm1, %xmm1
15557        vpxor	%xmm2, %xmm5, %xmm5
15558        vpclmulqdq	$16, %xmm0, %xmm1, %xmm2
15559        vpxor	%xmm3, %xmm5, %xmm5
15560        vpclmulqdq	$0x01, %xmm0, %xmm1, %xmm3
15561        vpxor	%xmm4, %xmm6, %xmm6
15562        vpclmulqdq	$0x00, %xmm0, %xmm1, %xmm4
15563        vpclmulqdq	$0x11, %xmm0, %xmm1, %xmm1
15564        vmovdqu	128(%rdi), %xmm0
15565        vpxor	%xmm1, %xmm7, %xmm7
15566        vaesenc	%xmm0, %xmm8, %xmm8
15567        vaesenc	%xmm0, %xmm9, %xmm9
15568        vaesenc	%xmm0, %xmm10, %xmm10
15569        vaesenc	%xmm0, %xmm11, %xmm11
15570        vaesenc	%xmm0, %xmm12, %xmm12
15571        vaesenc	%xmm0, %xmm13, %xmm13
15572        vaesenc	%xmm0, %xmm14, %xmm14
15573        vaesenc	%xmm0, %xmm15, %xmm15
15574        # aesenc_pclmul_l
15575        vpxor	%xmm2, %xmm5, %xmm5
15576        vpxor	%xmm4, %xmm6, %xmm6
15577        vpxor	%xmm3, %xmm5, %xmm5
15578        vpslldq	$8, %xmm5, %xmm1
15579        vpsrldq	$8, %xmm5, %xmm5
15580        vmovdqu	144(%rdi), %xmm4
15581        vmovdqu	L_avx2_aes_gcm_mod2_128(%rip), %xmm0
15582        vaesenc	%xmm4, %xmm8, %xmm8
15583        vpxor	%xmm1, %xmm6, %xmm6
15584        vpxor	%xmm5, %xmm7, %xmm7
15585        vpclmulqdq	$16, %xmm0, %xmm6, %xmm3
15586        vaesenc	%xmm4, %xmm9, %xmm9
15587        vaesenc	%xmm4, %xmm10, %xmm10
15588        vaesenc	%xmm4, %xmm11, %xmm11
15589        vpshufd	$0x4e, %xmm6, %xmm6
15590        vpxor	%xmm3, %xmm6, %xmm6
15591        vpclmulqdq	$16, %xmm0, %xmm6, %xmm3
15592        vaesenc	%xmm4, %xmm12, %xmm12
15593        vaesenc	%xmm4, %xmm13, %xmm13
15594        vaesenc	%xmm4, %xmm14, %xmm14
15595        vpshufd	$0x4e, %xmm6, %xmm6
15596        vpxor	%xmm3, %xmm6, %xmm6
15597        vpxor	%xmm7, %xmm6, %xmm6
15598        vaesenc	%xmm4, %xmm15, %xmm15
15599        cmpl	$11, %esi
15600        vmovdqu	160(%rdi), %xmm7
15601        jl	L_AES_GCM_encrypt_update_avx2_aesenc_128_ghash_avx_done
15602        vaesenc	%xmm7, %xmm8, %xmm8
15603        vaesenc	%xmm7, %xmm9, %xmm9
15604        vaesenc	%xmm7, %xmm10, %xmm10
15605        vaesenc	%xmm7, %xmm11, %xmm11
15606        vaesenc	%xmm7, %xmm12, %xmm12
15607        vaesenc	%xmm7, %xmm13, %xmm13
15608        vaesenc	%xmm7, %xmm14, %xmm14
15609        vaesenc	%xmm7, %xmm15, %xmm15
15610        vmovdqu	176(%rdi), %xmm7
15611        vaesenc	%xmm7, %xmm8, %xmm8
15612        vaesenc	%xmm7, %xmm9, %xmm9
15613        vaesenc	%xmm7, %xmm10, %xmm10
15614        vaesenc	%xmm7, %xmm11, %xmm11
15615        vaesenc	%xmm7, %xmm12, %xmm12
15616        vaesenc	%xmm7, %xmm13, %xmm13
15617        vaesenc	%xmm7, %xmm14, %xmm14
15618        vaesenc	%xmm7, %xmm15, %xmm15
15619        cmpl	$13, %esi
15620        vmovdqu	192(%rdi), %xmm7
15621        jl	L_AES_GCM_encrypt_update_avx2_aesenc_128_ghash_avx_done
15622        vaesenc	%xmm7, %xmm8, %xmm8
15623        vaesenc	%xmm7, %xmm9, %xmm9
15624        vaesenc	%xmm7, %xmm10, %xmm10
15625        vaesenc	%xmm7, %xmm11, %xmm11
15626        vaesenc	%xmm7, %xmm12, %xmm12
15627        vaesenc	%xmm7, %xmm13, %xmm13
15628        vaesenc	%xmm7, %xmm14, %xmm14
15629        vaesenc	%xmm7, %xmm15, %xmm15
15630        vmovdqu	208(%rdi), %xmm7
15631        vaesenc	%xmm7, %xmm8, %xmm8
15632        vaesenc	%xmm7, %xmm9, %xmm9
15633        vaesenc	%xmm7, %xmm10, %xmm10
15634        vaesenc	%xmm7, %xmm11, %xmm11
15635        vaesenc	%xmm7, %xmm12, %xmm12
15636        vaesenc	%xmm7, %xmm13, %xmm13
15637        vaesenc	%xmm7, %xmm14, %xmm14
15638        vaesenc	%xmm7, %xmm15, %xmm15
15639        vmovdqu	224(%rdi), %xmm7
15640L_AES_GCM_encrypt_update_avx2_aesenc_128_ghash_avx_done:
15641        # aesenc_last
15642        vaesenclast	%xmm7, %xmm8, %xmm8
15643        vaesenclast	%xmm7, %xmm9, %xmm9
15644        vaesenclast	%xmm7, %xmm10, %xmm10
15645        vaesenclast	%xmm7, %xmm11, %xmm11
15646        vmovdqu	(%rcx), %xmm0
15647        vmovdqu	16(%rcx), %xmm1
15648        vmovdqu	32(%rcx), %xmm2
15649        vmovdqu	48(%rcx), %xmm3
15650        vpxor	%xmm0, %xmm8, %xmm8
15651        vpxor	%xmm1, %xmm9, %xmm9
15652        vpxor	%xmm2, %xmm10, %xmm10
15653        vpxor	%xmm3, %xmm11, %xmm11
15654        vmovdqu	%xmm8, (%rdx)
15655        vmovdqu	%xmm9, 16(%rdx)
15656        vmovdqu	%xmm10, 32(%rdx)
15657        vmovdqu	%xmm11, 48(%rdx)
15658        vaesenclast	%xmm7, %xmm12, %xmm12
15659        vaesenclast	%xmm7, %xmm13, %xmm13
15660        vaesenclast	%xmm7, %xmm14, %xmm14
15661        vaesenclast	%xmm7, %xmm15, %xmm15
15662        vmovdqu	64(%rcx), %xmm0
15663        vmovdqu	80(%rcx), %xmm1
15664        vmovdqu	96(%rcx), %xmm2
15665        vmovdqu	112(%rcx), %xmm3
15666        vpxor	%xmm0, %xmm12, %xmm12
15667        vpxor	%xmm1, %xmm13, %xmm13
15668        vpxor	%xmm2, %xmm14, %xmm14
15669        vpxor	%xmm3, %xmm15, %xmm15
15670        vmovdqu	%xmm12, 64(%rdx)
15671        vmovdqu	%xmm13, 80(%rdx)
15672        vmovdqu	%xmm14, 96(%rdx)
15673        vmovdqu	%xmm15, 112(%rdx)
15674        # aesenc_128_ghash - end
15675        addl	$0x80, %r14d
15676        cmpl	%r13d, %r14d
15677        jl	L_AES_GCM_encrypt_update_avx2_ghash_128
15678L_AES_GCM_encrypt_update_avx2_end_128:
15679        vmovdqu	L_avx2_aes_gcm_bswap_mask(%rip), %xmm4
15680        vpshufb	%xmm4, %xmm8, %xmm8
15681        vpshufb	%xmm4, %xmm9, %xmm9
15682        vpshufb	%xmm4, %xmm10, %xmm10
15683        vpshufb	%xmm4, %xmm11, %xmm11
15684        vpshufb	%xmm4, %xmm12, %xmm12
15685        vpshufb	%xmm4, %xmm13, %xmm13
15686        vpshufb	%xmm4, %xmm14, %xmm14
15687        vpshufb	%xmm4, %xmm15, %xmm15
15688        vpxor	%xmm6, %xmm8, %xmm8
15689        vmovdqu	(%rsp), %xmm7
15690        vpclmulqdq	$16, %xmm15, %xmm7, %xmm5
15691        vpclmulqdq	$0x01, %xmm15, %xmm7, %xmm1
15692        vpclmulqdq	$0x00, %xmm15, %xmm7, %xmm4
15693        vpclmulqdq	$0x11, %xmm15, %xmm7, %xmm6
15694        vpxor	%xmm1, %xmm5, %xmm5
15695        vmovdqu	16(%rsp), %xmm7
15696        vpclmulqdq	$16, %xmm14, %xmm7, %xmm2
15697        vpclmulqdq	$0x01, %xmm14, %xmm7, %xmm1
15698        vpclmulqdq	$0x00, %xmm14, %xmm7, %xmm0
15699        vpclmulqdq	$0x11, %xmm14, %xmm7, %xmm3
15700        vpxor	%xmm1, %xmm2, %xmm2
15701        vpxor	%xmm3, %xmm6, %xmm6
15702        vpxor	%xmm2, %xmm5, %xmm5
15703        vpxor	%xmm0, %xmm4, %xmm4
15704        vmovdqu	32(%rsp), %xmm15
15705        vmovdqu	48(%rsp), %xmm7
15706        vpclmulqdq	$16, %xmm13, %xmm15, %xmm2
15707        vpclmulqdq	$0x01, %xmm13, %xmm15, %xmm1
15708        vpclmulqdq	$0x00, %xmm13, %xmm15, %xmm0
15709        vpclmulqdq	$0x11, %xmm13, %xmm15, %xmm3
15710        vpxor	%xmm1, %xmm2, %xmm2
15711        vpxor	%xmm3, %xmm6, %xmm6
15712        vpxor	%xmm2, %xmm5, %xmm5
15713        vpxor	%xmm0, %xmm4, %xmm4
15714        vpclmulqdq	$16, %xmm12, %xmm7, %xmm2
15715        vpclmulqdq	$0x01, %xmm12, %xmm7, %xmm1
15716        vpclmulqdq	$0x00, %xmm12, %xmm7, %xmm0
15717        vpclmulqdq	$0x11, %xmm12, %xmm7, %xmm3
15718        vpxor	%xmm1, %xmm2, %xmm2
15719        vpxor	%xmm3, %xmm6, %xmm6
15720        vpxor	%xmm2, %xmm5, %xmm5
15721        vpxor	%xmm0, %xmm4, %xmm4
15722        vmovdqu	64(%rsp), %xmm15
15723        vmovdqu	80(%rsp), %xmm7
15724        vpclmulqdq	$16, %xmm11, %xmm15, %xmm2
15725        vpclmulqdq	$0x01, %xmm11, %xmm15, %xmm1
15726        vpclmulqdq	$0x00, %xmm11, %xmm15, %xmm0
15727        vpclmulqdq	$0x11, %xmm11, %xmm15, %xmm3
15728        vpxor	%xmm1, %xmm2, %xmm2
15729        vpxor	%xmm3, %xmm6, %xmm6
15730        vpxor	%xmm2, %xmm5, %xmm5
15731        vpxor	%xmm0, %xmm4, %xmm4
15732        vpclmulqdq	$16, %xmm10, %xmm7, %xmm2
15733        vpclmulqdq	$0x01, %xmm10, %xmm7, %xmm1
15734        vpclmulqdq	$0x00, %xmm10, %xmm7, %xmm0
15735        vpclmulqdq	$0x11, %xmm10, %xmm7, %xmm3
15736        vpxor	%xmm1, %xmm2, %xmm2
15737        vpxor	%xmm3, %xmm6, %xmm6
15738        vpxor	%xmm2, %xmm5, %xmm5
15739        vpxor	%xmm0, %xmm4, %xmm4
15740        vmovdqu	96(%rsp), %xmm15
15741        vmovdqu	112(%rsp), %xmm7
15742        vpclmulqdq	$16, %xmm9, %xmm15, %xmm2
15743        vpclmulqdq	$0x01, %xmm9, %xmm15, %xmm1
15744        vpclmulqdq	$0x00, %xmm9, %xmm15, %xmm0
15745        vpclmulqdq	$0x11, %xmm9, %xmm15, %xmm3
15746        vpxor	%xmm1, %xmm2, %xmm2
15747        vpxor	%xmm3, %xmm6, %xmm6
15748        vpxor	%xmm2, %xmm5, %xmm5
15749        vpxor	%xmm0, %xmm4, %xmm4
15750        vpclmulqdq	$16, %xmm8, %xmm7, %xmm2
15751        vpclmulqdq	$0x01, %xmm8, %xmm7, %xmm1
15752        vpclmulqdq	$0x00, %xmm8, %xmm7, %xmm0
15753        vpclmulqdq	$0x11, %xmm8, %xmm7, %xmm3
15754        vpxor	%xmm1, %xmm2, %xmm2
15755        vpxor	%xmm3, %xmm6, %xmm6
15756        vpxor	%xmm2, %xmm5, %xmm5
15757        vpxor	%xmm0, %xmm4, %xmm4
15758        vpslldq	$8, %xmm5, %xmm7
15759        vpsrldq	$8, %xmm5, %xmm5
15760        vpxor	%xmm7, %xmm4, %xmm4
15761        vpxor	%xmm5, %xmm6, %xmm6
15762        # ghash_red
15763        vmovdqu	L_avx2_aes_gcm_mod2_128(%rip), %xmm2
15764        vpclmulqdq	$16, %xmm2, %xmm4, %xmm0
15765        vpshufd	$0x4e, %xmm4, %xmm1
15766        vpxor	%xmm0, %xmm1, %xmm1
15767        vpclmulqdq	$16, %xmm2, %xmm1, %xmm0
15768        vpshufd	$0x4e, %xmm1, %xmm1
15769        vpxor	%xmm0, %xmm1, %xmm1
15770        vpxor	%xmm1, %xmm6, %xmm6
15771        vmovdqu	(%rsp), %xmm5
15772        vmovdqu	128(%rsp), %xmm4
15773L_AES_GCM_encrypt_update_avx2_done_128:
15774        cmpl	%r8d, %r14d
15775        je	L_AES_GCM_encrypt_update_avx2_done_enc
15776        movl	%r8d, %r13d
15777        andl	$0xfffffff0, %r13d
15778        cmpl	%r13d, %r14d
15779        jge	L_AES_GCM_encrypt_update_avx2_last_block_done
15780        # aesenc_block
15781        vmovdqu	%xmm4, %xmm1
15782        vpshufb	L_avx2_aes_gcm_bswap_epi64(%rip), %xmm1, %xmm0
15783        vpaddd	L_avx2_aes_gcm_one(%rip), %xmm1, %xmm1
15784        vpxor	(%rdi), %xmm0, %xmm0
15785        vmovdqu	16(%rdi), %xmm2
15786        vaesenc	%xmm2, %xmm0, %xmm0
15787        vmovdqu	32(%rdi), %xmm2
15788        vaesenc	%xmm2, %xmm0, %xmm0
15789        vmovdqu	48(%rdi), %xmm2
15790        vaesenc	%xmm2, %xmm0, %xmm0
15791        vmovdqu	64(%rdi), %xmm2
15792        vaesenc	%xmm2, %xmm0, %xmm0
15793        vmovdqu	80(%rdi), %xmm2
15794        vaesenc	%xmm2, %xmm0, %xmm0
15795        vmovdqu	96(%rdi), %xmm2
15796        vaesenc	%xmm2, %xmm0, %xmm0
15797        vmovdqu	112(%rdi), %xmm2
15798        vaesenc	%xmm2, %xmm0, %xmm0
15799        vmovdqu	128(%rdi), %xmm2
15800        vaesenc	%xmm2, %xmm0, %xmm0
15801        vmovdqu	144(%rdi), %xmm2
15802        vaesenc	%xmm2, %xmm0, %xmm0
15803        vmovdqu	%xmm1, %xmm4
15804        cmpl	$11, %esi
15805        vmovdqu	160(%rdi), %xmm1
15806        jl	L_AES_GCM_encrypt_update_avx2_aesenc_block_last
15807        vaesenc	%xmm1, %xmm0, %xmm0
15808        vmovdqu	176(%rdi), %xmm2
15809        vaesenc	%xmm2, %xmm0, %xmm0
15810        cmpl	$13, %esi
15811        vmovdqu	192(%rdi), %xmm1
15812        jl	L_AES_GCM_encrypt_update_avx2_aesenc_block_last
15813        vaesenc	%xmm1, %xmm0, %xmm0
15814        vmovdqu	208(%rdi), %xmm2
15815        vaesenc	%xmm2, %xmm0, %xmm0
15816        vmovdqu	224(%rdi), %xmm1
15817L_AES_GCM_encrypt_update_avx2_aesenc_block_last:
15818        vaesenclast	%xmm1, %xmm0, %xmm0
15819        vmovdqu	(%r11,%r14,1), %xmm1
15820        vpxor	%xmm1, %xmm0, %xmm0
15821        vmovdqu	%xmm0, (%r10,%r14,1)
15822        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0
15823        vpxor	%xmm0, %xmm6, %xmm6
15824        addl	$16, %r14d
15825        cmpl	%r13d, %r14d
15826        jge	L_AES_GCM_encrypt_update_avx2_last_block_ghash
15827L_AES_GCM_encrypt_update_avx2_last_block_start:
15828        vmovdqu	(%r11,%r14,1), %xmm12
15829        vpshufb	L_avx2_aes_gcm_bswap_epi64(%rip), %xmm4, %xmm11
15830        vpaddd	L_avx2_aes_gcm_one(%rip), %xmm4, %xmm4
15831        # aesenc_gfmul_sb
15832        vpclmulqdq	$0x01, %xmm5, %xmm6, %xmm2
15833        vpclmulqdq	$16, %xmm5, %xmm6, %xmm3
15834        vpclmulqdq	$0x00, %xmm5, %xmm6, %xmm1
15835        vpclmulqdq	$0x11, %xmm5, %xmm6, %xmm8
15836        vpxor	(%rdi), %xmm11, %xmm11
15837        vaesenc	16(%rdi), %xmm11, %xmm11
15838        vpxor	%xmm2, %xmm3, %xmm3
15839        vpslldq	$8, %xmm3, %xmm2
15840        vpsrldq	$8, %xmm3, %xmm3
15841        vaesenc	32(%rdi), %xmm11, %xmm11
15842        vpxor	%xmm1, %xmm2, %xmm2
15843        vpclmulqdq	$16, L_avx2_aes_gcm_mod2_128(%rip), %xmm2, %xmm1
15844        vaesenc	48(%rdi), %xmm11, %xmm11
15845        vaesenc	64(%rdi), %xmm11, %xmm11
15846        vaesenc	80(%rdi), %xmm11, %xmm11
15847        vpshufd	$0x4e, %xmm2, %xmm2
15848        vpxor	%xmm1, %xmm2, %xmm2
15849        vpclmulqdq	$16, L_avx2_aes_gcm_mod2_128(%rip), %xmm2, %xmm1
15850        vaesenc	96(%rdi), %xmm11, %xmm11
15851        vaesenc	112(%rdi), %xmm11, %xmm11
15852        vaesenc	128(%rdi), %xmm11, %xmm11
15853        vpshufd	$0x4e, %xmm2, %xmm2
15854        vaesenc	144(%rdi), %xmm11, %xmm11
15855        vpxor	%xmm3, %xmm8, %xmm8
15856        vpxor	%xmm8, %xmm2, %xmm2
15857        vmovdqu	160(%rdi), %xmm0
15858        cmpl	$11, %esi
15859        jl	L_AES_GCM_encrypt_update_avx2_aesenc_gfmul_sb_last
15860        vaesenc	%xmm0, %xmm11, %xmm11
15861        vaesenc	176(%rdi), %xmm11, %xmm11
15862        vmovdqu	192(%rdi), %xmm0
15863        cmpl	$13, %esi
15864        jl	L_AES_GCM_encrypt_update_avx2_aesenc_gfmul_sb_last
15865        vaesenc	%xmm0, %xmm11, %xmm11
15866        vaesenc	208(%rdi), %xmm11, %xmm11
15867        vmovdqu	224(%rdi), %xmm0
15868L_AES_GCM_encrypt_update_avx2_aesenc_gfmul_sb_last:
15869        vaesenclast	%xmm0, %xmm11, %xmm11
15870        vpxor	%xmm1, %xmm2, %xmm6
15871        vpxor	%xmm12, %xmm11, %xmm11
15872        vmovdqu	%xmm11, (%r10,%r14,1)
15873        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm11, %xmm11
15874        vpxor	%xmm11, %xmm6, %xmm6
15875        addl	$16, %r14d
15876        cmpl	%r13d, %r14d
15877        jl	L_AES_GCM_encrypt_update_avx2_last_block_start
15878L_AES_GCM_encrypt_update_avx2_last_block_ghash:
15879        # ghash_gfmul_red
15880        vpclmulqdq	$16, %xmm5, %xmm6, %xmm10
15881        vpclmulqdq	$0x01, %xmm5, %xmm6, %xmm9
15882        vpclmulqdq	$0x00, %xmm5, %xmm6, %xmm8
15883        vpxor	%xmm9, %xmm10, %xmm10
15884        vpslldq	$8, %xmm10, %xmm9
15885        vpsrldq	$8, %xmm10, %xmm10
15886        vpxor	%xmm8, %xmm9, %xmm9
15887        vpclmulqdq	$0x11, %xmm5, %xmm6, %xmm6
15888        vpclmulqdq	$16, L_avx2_aes_gcm_mod2_128(%rip), %xmm9, %xmm8
15889        vpshufd	$0x4e, %xmm9, %xmm9
15890        vpxor	%xmm8, %xmm9, %xmm9
15891        vpclmulqdq	$16, L_avx2_aes_gcm_mod2_128(%rip), %xmm9, %xmm8
15892        vpshufd	$0x4e, %xmm9, %xmm9
15893        vpxor	%xmm10, %xmm6, %xmm6
15894        vpxor	%xmm9, %xmm6, %xmm6
15895        vpxor	%xmm8, %xmm6, %xmm6
15896L_AES_GCM_encrypt_update_avx2_last_block_done:
15897L_AES_GCM_encrypt_update_avx2_done_enc:
15898        vmovdqu	%xmm6, (%r9)
15899        vmovdqu	%xmm4, (%r12)
15900        vzeroupper
15901        addq	$0x98, %rsp
15902        popq	%r14
15903        popq	%r13
15904        popq	%r12
15905        repz retq
15906#ifndef __APPLE__
15907.size	AES_GCM_encrypt_update_avx2,.-AES_GCM_encrypt_update_avx2
15908#endif /* __APPLE__ */
15909#ifndef __APPLE__
15910.text
15911.globl	AES_GCM_encrypt_final_avx2
15912.type	AES_GCM_encrypt_final_avx2,@function
15913.align	16
15914AES_GCM_encrypt_final_avx2:
15915#else
15916.section	__TEXT,__text
15917.globl	_AES_GCM_encrypt_final_avx2
15918.p2align	4
15919_AES_GCM_encrypt_final_avx2:
15920#endif /* __APPLE__ */
15921        movq	8(%rsp), %rax
15922        subq	$16, %rsp
15923        vmovdqu	(%rdi), %xmm4
15924        vmovdqu	(%r9), %xmm5
15925        vmovdqu	(%rax), %xmm6
15926        vpsrlq	$63, %xmm5, %xmm1
15927        vpsllq	$0x01, %xmm5, %xmm0
15928        vpslldq	$8, %xmm1, %xmm1
15929        vpor	%xmm1, %xmm0, %xmm0
15930        vpshufd	$0xff, %xmm5, %xmm5
15931        vpsrad	$31, %xmm5, %xmm5
15932        vpand	L_avx2_aes_gcm_mod2_128(%rip), %xmm5, %xmm5
15933        vpxor	%xmm0, %xmm5, %xmm5
15934        # calc_tag
15935        shlq	$3, %rcx
15936        shlq	$3, %r8
15937        vmovq	%rcx, %xmm0
15938        vmovq	%r8, %xmm1
15939        vpunpcklqdq	%xmm1, %xmm0, %xmm0
15940        vpxor	%xmm4, %xmm0, %xmm0
15941        # ghash_gfmul_red
15942        vpclmulqdq	$16, %xmm5, %xmm0, %xmm7
15943        vpclmulqdq	$0x01, %xmm5, %xmm0, %xmm3
15944        vpclmulqdq	$0x00, %xmm5, %xmm0, %xmm2
15945        vpxor	%xmm3, %xmm7, %xmm7
15946        vpslldq	$8, %xmm7, %xmm3
15947        vpsrldq	$8, %xmm7, %xmm7
15948        vpxor	%xmm2, %xmm3, %xmm3
15949        vpclmulqdq	$0x11, %xmm5, %xmm0, %xmm0
15950        vpclmulqdq	$16, L_avx2_aes_gcm_mod2_128(%rip), %xmm3, %xmm2
15951        vpshufd	$0x4e, %xmm3, %xmm3
15952        vpxor	%xmm2, %xmm3, %xmm3
15953        vpclmulqdq	$16, L_avx2_aes_gcm_mod2_128(%rip), %xmm3, %xmm2
15954        vpshufd	$0x4e, %xmm3, %xmm3
15955        vpxor	%xmm7, %xmm0, %xmm0
15956        vpxor	%xmm3, %xmm0, %xmm0
15957        vpxor	%xmm2, %xmm0, %xmm0
15958        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0
15959        vpxor	%xmm6, %xmm0, %xmm0
15960        # store_tag
15961        cmpl	$16, %edx
15962        je	L_AES_GCM_encrypt_final_avx2_store_tag_16
15963        xorq	%r10, %r10
15964        vmovdqu	%xmm0, (%rsp)
15965L_AES_GCM_encrypt_final_avx2_store_tag_loop:
15966        movzbl	(%rsp,%r10,1), %r11d
15967        movb	%r11b, (%rsi,%r10,1)
15968        incl	%r10d
15969        cmpl	%edx, %r10d
15970        jne	L_AES_GCM_encrypt_final_avx2_store_tag_loop
15971        jmp	L_AES_GCM_encrypt_final_avx2_store_tag_done
15972L_AES_GCM_encrypt_final_avx2_store_tag_16:
15973        vmovdqu	%xmm0, (%rsi)
15974L_AES_GCM_encrypt_final_avx2_store_tag_done:
15975        vzeroupper
15976        addq	$16, %rsp
15977        repz retq
15978#ifndef __APPLE__
15979.size	AES_GCM_encrypt_final_avx2,.-AES_GCM_encrypt_final_avx2
15980#endif /* __APPLE__ */
15981#ifndef __APPLE__
15982.text
15983.globl	AES_GCM_decrypt_update_avx2
15984.type	AES_GCM_decrypt_update_avx2,@function
15985.align	16
15986AES_GCM_decrypt_update_avx2:
15987#else
15988.section	__TEXT,__text
15989.globl	_AES_GCM_decrypt_update_avx2
15990.p2align	4
15991_AES_GCM_decrypt_update_avx2:
15992#endif /* __APPLE__ */
15993        pushq	%r13
15994        pushq	%r12
15995        pushq	%r14
15996        movq	%rdx, %r10
15997        movq	%rcx, %r11
15998        movq	32(%rsp), %rax
15999        movq	40(%rsp), %r12
16000        subq	$0xa8, %rsp
16001        vmovdqu	(%r9), %xmm6
16002        vmovdqu	(%rax), %xmm5
16003        vmovdqu	(%r12), %xmm4
16004        # Calculate H
16005        vpsrlq	$63, %xmm5, %xmm1
16006        vpsllq	$0x01, %xmm5, %xmm0
16007        vpslldq	$8, %xmm1, %xmm1
16008        vpor	%xmm1, %xmm0, %xmm0
16009        vpshufd	$0xff, %xmm5, %xmm5
16010        vpsrad	$31, %xmm5, %xmm5
16011        vpand	L_avx2_aes_gcm_mod2_128(%rip), %xmm5, %xmm5
16012        vpxor	%xmm0, %xmm5, %xmm5
16013        xorl	%r14d, %r14d
16014        cmpl	$0x80, %r8d
16015        movl	%r8d, %r13d
16016        jl	L_AES_GCM_decrypt_update_avx2_done_128
16017        andl	$0xffffff80, %r13d
16018        vmovdqu	%xmm4, 128(%rsp)
16019        vmovdqu	%xmm15, 144(%rsp)
16020        vmovdqu	L_avx2_aes_gcm_mod2_128(%rip), %xmm3
16021        # H ^ 1 and H ^ 2
16022        vpclmulqdq	$0x00, %xmm5, %xmm5, %xmm9
16023        vpclmulqdq	$0x11, %xmm5, %xmm5, %xmm10
16024        vpclmulqdq	$16, %xmm3, %xmm9, %xmm8
16025        vpshufd	$0x4e, %xmm9, %xmm9
16026        vpxor	%xmm8, %xmm9, %xmm9
16027        vpclmulqdq	$16, %xmm3, %xmm9, %xmm8
16028        vpshufd	$0x4e, %xmm9, %xmm9
16029        vpxor	%xmm8, %xmm9, %xmm9
16030        vpxor	%xmm9, %xmm10, %xmm0
16031        vmovdqu	%xmm5, (%rsp)
16032        vmovdqu	%xmm0, 16(%rsp)
16033        # H ^ 3 and H ^ 4
16034        vpclmulqdq	$16, %xmm5, %xmm0, %xmm11
16035        vpclmulqdq	$0x01, %xmm5, %xmm0, %xmm10
16036        vpclmulqdq	$0x00, %xmm5, %xmm0, %xmm9
16037        vpclmulqdq	$0x11, %xmm5, %xmm0, %xmm12
16038        vpclmulqdq	$0x00, %xmm0, %xmm0, %xmm13
16039        vpclmulqdq	$0x11, %xmm0, %xmm0, %xmm14
16040        vpxor	%xmm10, %xmm11, %xmm11
16041        vpslldq	$8, %xmm11, %xmm10
16042        vpsrldq	$8, %xmm11, %xmm11
16043        vpxor	%xmm9, %xmm10, %xmm10
16044        vpclmulqdq	$16, %xmm3, %xmm13, %xmm8
16045        vpclmulqdq	$16, %xmm3, %xmm10, %xmm9
16046        vpshufd	$0x4e, %xmm10, %xmm10
16047        vpshufd	$0x4e, %xmm13, %xmm13
16048        vpxor	%xmm9, %xmm10, %xmm10
16049        vpxor	%xmm8, %xmm13, %xmm13
16050        vpclmulqdq	$16, %xmm3, %xmm10, %xmm9
16051        vpclmulqdq	$16, %xmm3, %xmm13, %xmm8
16052        vpshufd	$0x4e, %xmm10, %xmm10
16053        vpshufd	$0x4e, %xmm13, %xmm13
16054        vpxor	%xmm11, %xmm12, %xmm12
16055        vpxor	%xmm8, %xmm13, %xmm13
16056        vpxor	%xmm12, %xmm10, %xmm10
16057        vpxor	%xmm14, %xmm13, %xmm2
16058        vpxor	%xmm9, %xmm10, %xmm1
16059        vmovdqu	%xmm1, 32(%rsp)
16060        vmovdqu	%xmm2, 48(%rsp)
16061        # H ^ 5 and H ^ 6
16062        vpclmulqdq	$16, %xmm0, %xmm1, %xmm11
16063        vpclmulqdq	$0x01, %xmm0, %xmm1, %xmm10
16064        vpclmulqdq	$0x00, %xmm0, %xmm1, %xmm9
16065        vpclmulqdq	$0x11, %xmm0, %xmm1, %xmm12
16066        vpclmulqdq	$0x00, %xmm1, %xmm1, %xmm13
16067        vpclmulqdq	$0x11, %xmm1, %xmm1, %xmm14
16068        vpxor	%xmm10, %xmm11, %xmm11
16069        vpslldq	$8, %xmm11, %xmm10
16070        vpsrldq	$8, %xmm11, %xmm11
16071        vpxor	%xmm9, %xmm10, %xmm10
16072        vpclmulqdq	$16, %xmm3, %xmm13, %xmm8
16073        vpclmulqdq	$16, %xmm3, %xmm10, %xmm9
16074        vpshufd	$0x4e, %xmm10, %xmm10
16075        vpshufd	$0x4e, %xmm13, %xmm13
16076        vpxor	%xmm9, %xmm10, %xmm10
16077        vpxor	%xmm8, %xmm13, %xmm13
16078        vpclmulqdq	$16, %xmm3, %xmm10, %xmm9
16079        vpclmulqdq	$16, %xmm3, %xmm13, %xmm8
16080        vpshufd	$0x4e, %xmm10, %xmm10
16081        vpshufd	$0x4e, %xmm13, %xmm13
16082        vpxor	%xmm11, %xmm12, %xmm12
16083        vpxor	%xmm8, %xmm13, %xmm13
16084        vpxor	%xmm12, %xmm10, %xmm10
16085        vpxor	%xmm14, %xmm13, %xmm0
16086        vpxor	%xmm9, %xmm10, %xmm7
16087        vmovdqu	%xmm7, 64(%rsp)
16088        vmovdqu	%xmm0, 80(%rsp)
16089        # H ^ 7 and H ^ 8
16090        vpclmulqdq	$16, %xmm1, %xmm2, %xmm11
16091        vpclmulqdq	$0x01, %xmm1, %xmm2, %xmm10
16092        vpclmulqdq	$0x00, %xmm1, %xmm2, %xmm9
16093        vpclmulqdq	$0x11, %xmm1, %xmm2, %xmm12
16094        vpclmulqdq	$0x00, %xmm2, %xmm2, %xmm13
16095        vpclmulqdq	$0x11, %xmm2, %xmm2, %xmm14
16096        vpxor	%xmm10, %xmm11, %xmm11
16097        vpslldq	$8, %xmm11, %xmm10
16098        vpsrldq	$8, %xmm11, %xmm11
16099        vpxor	%xmm9, %xmm10, %xmm10
16100        vpclmulqdq	$16, %xmm3, %xmm13, %xmm8
16101        vpclmulqdq	$16, %xmm3, %xmm10, %xmm9
16102        vpshufd	$0x4e, %xmm10, %xmm10
16103        vpshufd	$0x4e, %xmm13, %xmm13
16104        vpxor	%xmm9, %xmm10, %xmm10
16105        vpxor	%xmm8, %xmm13, %xmm13
16106        vpclmulqdq	$16, %xmm3, %xmm10, %xmm9
16107        vpclmulqdq	$16, %xmm3, %xmm13, %xmm8
16108        vpshufd	$0x4e, %xmm10, %xmm10
16109        vpshufd	$0x4e, %xmm13, %xmm13
16110        vpxor	%xmm11, %xmm12, %xmm12
16111        vpxor	%xmm8, %xmm13, %xmm13
16112        vpxor	%xmm12, %xmm10, %xmm10
16113        vpxor	%xmm14, %xmm13, %xmm0
16114        vpxor	%xmm9, %xmm10, %xmm7
16115        vmovdqu	%xmm7, 96(%rsp)
16116        vmovdqu	%xmm0, 112(%rsp)
16117L_AES_GCM_decrypt_update_avx2_ghash_128:
16118        # aesenc_128_ghash
16119        leaq	(%r11,%r14,1), %rcx
16120        leaq	(%r10,%r14,1), %rdx
16121        # aesenc_ctr
16122        vmovdqu	128(%rsp), %xmm0
16123        vmovdqu	L_avx2_aes_gcm_bswap_epi64(%rip), %xmm1
16124        vpaddd	L_avx2_aes_gcm_one(%rip), %xmm0, %xmm9
16125        vpshufb	%xmm1, %xmm0, %xmm8
16126        vpaddd	L_avx2_aes_gcm_two(%rip), %xmm0, %xmm10
16127        vpshufb	%xmm1, %xmm9, %xmm9
16128        vpaddd	L_avx2_aes_gcm_three(%rip), %xmm0, %xmm11
16129        vpshufb	%xmm1, %xmm10, %xmm10
16130        vpaddd	L_avx2_aes_gcm_four(%rip), %xmm0, %xmm12
16131        vpshufb	%xmm1, %xmm11, %xmm11
16132        vpaddd	L_avx2_aes_gcm_five(%rip), %xmm0, %xmm13
16133        vpshufb	%xmm1, %xmm12, %xmm12
16134        vpaddd	L_avx2_aes_gcm_six(%rip), %xmm0, %xmm14
16135        vpshufb	%xmm1, %xmm13, %xmm13
16136        vpaddd	L_avx2_aes_gcm_seven(%rip), %xmm0, %xmm15
16137        vpshufb	%xmm1, %xmm14, %xmm14
16138        vpaddd	L_avx2_aes_gcm_eight(%rip), %xmm0, %xmm0
16139        vpshufb	%xmm1, %xmm15, %xmm15
16140        # aesenc_xor
16141        vmovdqu	(%rdi), %xmm7
16142        vmovdqu	%xmm0, 128(%rsp)
16143        vpxor	%xmm7, %xmm8, %xmm8
16144        vpxor	%xmm7, %xmm9, %xmm9
16145        vpxor	%xmm7, %xmm10, %xmm10
16146        vpxor	%xmm7, %xmm11, %xmm11
16147        vpxor	%xmm7, %xmm12, %xmm12
16148        vpxor	%xmm7, %xmm13, %xmm13
16149        vpxor	%xmm7, %xmm14, %xmm14
16150        vpxor	%xmm7, %xmm15, %xmm15
16151        # aesenc_pclmul_1
16152        vmovdqu	(%rcx), %xmm1
16153        vmovdqu	16(%rdi), %xmm0
16154        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm1, %xmm1
16155        vmovdqu	112(%rsp), %xmm2
16156        vpxor	%xmm6, %xmm1, %xmm1
16157        vpclmulqdq	$16, %xmm2, %xmm1, %xmm5
16158        vpclmulqdq	$0x01, %xmm2, %xmm1, %xmm3
16159        vpclmulqdq	$0x00, %xmm2, %xmm1, %xmm6
16160        vpclmulqdq	$0x11, %xmm2, %xmm1, %xmm7
16161        vaesenc	%xmm0, %xmm8, %xmm8
16162        vaesenc	%xmm0, %xmm9, %xmm9
16163        vaesenc	%xmm0, %xmm10, %xmm10
16164        vaesenc	%xmm0, %xmm11, %xmm11
16165        vaesenc	%xmm0, %xmm12, %xmm12
16166        vaesenc	%xmm0, %xmm13, %xmm13
16167        vaesenc	%xmm0, %xmm14, %xmm14
16168        vaesenc	%xmm0, %xmm15, %xmm15
16169        # aesenc_pclmul_2
16170        vmovdqu	16(%rcx), %xmm1
16171        vmovdqu	96(%rsp), %xmm0
16172        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm1, %xmm1
16173        vpxor	%xmm3, %xmm5, %xmm5
16174        vpclmulqdq	$16, %xmm0, %xmm1, %xmm2
16175        vpclmulqdq	$0x01, %xmm0, %xmm1, %xmm3
16176        vpclmulqdq	$0x00, %xmm0, %xmm1, %xmm4
16177        vpclmulqdq	$0x11, %xmm0, %xmm1, %xmm1
16178        vmovdqu	32(%rdi), %xmm0
16179        vpxor	%xmm1, %xmm7, %xmm7
16180        vaesenc	%xmm0, %xmm8, %xmm8
16181        vaesenc	%xmm0, %xmm9, %xmm9
16182        vaesenc	%xmm0, %xmm10, %xmm10
16183        vaesenc	%xmm0, %xmm11, %xmm11
16184        vaesenc	%xmm0, %xmm12, %xmm12
16185        vaesenc	%xmm0, %xmm13, %xmm13
16186        vaesenc	%xmm0, %xmm14, %xmm14
16187        vaesenc	%xmm0, %xmm15, %xmm15
16188        # aesenc_pclmul_n
16189        vmovdqu	32(%rcx), %xmm1
16190        vmovdqu	80(%rsp), %xmm0
16191        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm1, %xmm1
16192        vpxor	%xmm2, %xmm5, %xmm5
16193        vpclmulqdq	$16, %xmm0, %xmm1, %xmm2
16194        vpxor	%xmm3, %xmm5, %xmm5
16195        vpclmulqdq	$0x01, %xmm0, %xmm1, %xmm3
16196        vpxor	%xmm4, %xmm6, %xmm6
16197        vpclmulqdq	$0x00, %xmm0, %xmm1, %xmm4
16198        vpclmulqdq	$0x11, %xmm0, %xmm1, %xmm1
16199        vmovdqu	48(%rdi), %xmm0
16200        vpxor	%xmm1, %xmm7, %xmm7
16201        vaesenc	%xmm0, %xmm8, %xmm8
16202        vaesenc	%xmm0, %xmm9, %xmm9
16203        vaesenc	%xmm0, %xmm10, %xmm10
16204        vaesenc	%xmm0, %xmm11, %xmm11
16205        vaesenc	%xmm0, %xmm12, %xmm12
16206        vaesenc	%xmm0, %xmm13, %xmm13
16207        vaesenc	%xmm0, %xmm14, %xmm14
16208        vaesenc	%xmm0, %xmm15, %xmm15
16209        # aesenc_pclmul_n
16210        vmovdqu	48(%rcx), %xmm1
16211        vmovdqu	64(%rsp), %xmm0
16212        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm1, %xmm1
16213        vpxor	%xmm2, %xmm5, %xmm5
16214        vpclmulqdq	$16, %xmm0, %xmm1, %xmm2
16215        vpxor	%xmm3, %xmm5, %xmm5
16216        vpclmulqdq	$0x01, %xmm0, %xmm1, %xmm3
16217        vpxor	%xmm4, %xmm6, %xmm6
16218        vpclmulqdq	$0x00, %xmm0, %xmm1, %xmm4
16219        vpclmulqdq	$0x11, %xmm0, %xmm1, %xmm1
16220        vmovdqu	64(%rdi), %xmm0
16221        vpxor	%xmm1, %xmm7, %xmm7
16222        vaesenc	%xmm0, %xmm8, %xmm8
16223        vaesenc	%xmm0, %xmm9, %xmm9
16224        vaesenc	%xmm0, %xmm10, %xmm10
16225        vaesenc	%xmm0, %xmm11, %xmm11
16226        vaesenc	%xmm0, %xmm12, %xmm12
16227        vaesenc	%xmm0, %xmm13, %xmm13
16228        vaesenc	%xmm0, %xmm14, %xmm14
16229        vaesenc	%xmm0, %xmm15, %xmm15
16230        # aesenc_pclmul_n
16231        vmovdqu	64(%rcx), %xmm1
16232        vmovdqu	48(%rsp), %xmm0
16233        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm1, %xmm1
16234        vpxor	%xmm2, %xmm5, %xmm5
16235        vpclmulqdq	$16, %xmm0, %xmm1, %xmm2
16236        vpxor	%xmm3, %xmm5, %xmm5
16237        vpclmulqdq	$0x01, %xmm0, %xmm1, %xmm3
16238        vpxor	%xmm4, %xmm6, %xmm6
16239        vpclmulqdq	$0x00, %xmm0, %xmm1, %xmm4
16240        vpclmulqdq	$0x11, %xmm0, %xmm1, %xmm1
16241        vmovdqu	80(%rdi), %xmm0
16242        vpxor	%xmm1, %xmm7, %xmm7
16243        vaesenc	%xmm0, %xmm8, %xmm8
16244        vaesenc	%xmm0, %xmm9, %xmm9
16245        vaesenc	%xmm0, %xmm10, %xmm10
16246        vaesenc	%xmm0, %xmm11, %xmm11
16247        vaesenc	%xmm0, %xmm12, %xmm12
16248        vaesenc	%xmm0, %xmm13, %xmm13
16249        vaesenc	%xmm0, %xmm14, %xmm14
16250        vaesenc	%xmm0, %xmm15, %xmm15
16251        # aesenc_pclmul_n
16252        vmovdqu	80(%rcx), %xmm1
16253        vmovdqu	32(%rsp), %xmm0
16254        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm1, %xmm1
16255        vpxor	%xmm2, %xmm5, %xmm5
16256        vpclmulqdq	$16, %xmm0, %xmm1, %xmm2
16257        vpxor	%xmm3, %xmm5, %xmm5
16258        vpclmulqdq	$0x01, %xmm0, %xmm1, %xmm3
16259        vpxor	%xmm4, %xmm6, %xmm6
16260        vpclmulqdq	$0x00, %xmm0, %xmm1, %xmm4
16261        vpclmulqdq	$0x11, %xmm0, %xmm1, %xmm1
16262        vmovdqu	96(%rdi), %xmm0
16263        vpxor	%xmm1, %xmm7, %xmm7
16264        vaesenc	%xmm0, %xmm8, %xmm8
16265        vaesenc	%xmm0, %xmm9, %xmm9
16266        vaesenc	%xmm0, %xmm10, %xmm10
16267        vaesenc	%xmm0, %xmm11, %xmm11
16268        vaesenc	%xmm0, %xmm12, %xmm12
16269        vaesenc	%xmm0, %xmm13, %xmm13
16270        vaesenc	%xmm0, %xmm14, %xmm14
16271        vaesenc	%xmm0, %xmm15, %xmm15
16272        # aesenc_pclmul_n
16273        vmovdqu	96(%rcx), %xmm1
16274        vmovdqu	16(%rsp), %xmm0
16275        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm1, %xmm1
16276        vpxor	%xmm2, %xmm5, %xmm5
16277        vpclmulqdq	$16, %xmm0, %xmm1, %xmm2
16278        vpxor	%xmm3, %xmm5, %xmm5
16279        vpclmulqdq	$0x01, %xmm0, %xmm1, %xmm3
16280        vpxor	%xmm4, %xmm6, %xmm6
16281        vpclmulqdq	$0x00, %xmm0, %xmm1, %xmm4
16282        vpclmulqdq	$0x11, %xmm0, %xmm1, %xmm1
16283        vmovdqu	112(%rdi), %xmm0
16284        vpxor	%xmm1, %xmm7, %xmm7
16285        vaesenc	%xmm0, %xmm8, %xmm8
16286        vaesenc	%xmm0, %xmm9, %xmm9
16287        vaesenc	%xmm0, %xmm10, %xmm10
16288        vaesenc	%xmm0, %xmm11, %xmm11
16289        vaesenc	%xmm0, %xmm12, %xmm12
16290        vaesenc	%xmm0, %xmm13, %xmm13
16291        vaesenc	%xmm0, %xmm14, %xmm14
16292        vaesenc	%xmm0, %xmm15, %xmm15
16293        # aesenc_pclmul_n
16294        vmovdqu	112(%rcx), %xmm1
16295        vmovdqu	(%rsp), %xmm0
16296        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm1, %xmm1
16297        vpxor	%xmm2, %xmm5, %xmm5
16298        vpclmulqdq	$16, %xmm0, %xmm1, %xmm2
16299        vpxor	%xmm3, %xmm5, %xmm5
16300        vpclmulqdq	$0x01, %xmm0, %xmm1, %xmm3
16301        vpxor	%xmm4, %xmm6, %xmm6
16302        vpclmulqdq	$0x00, %xmm0, %xmm1, %xmm4
16303        vpclmulqdq	$0x11, %xmm0, %xmm1, %xmm1
16304        vmovdqu	128(%rdi), %xmm0
16305        vpxor	%xmm1, %xmm7, %xmm7
16306        vaesenc	%xmm0, %xmm8, %xmm8
16307        vaesenc	%xmm0, %xmm9, %xmm9
16308        vaesenc	%xmm0, %xmm10, %xmm10
16309        vaesenc	%xmm0, %xmm11, %xmm11
16310        vaesenc	%xmm0, %xmm12, %xmm12
16311        vaesenc	%xmm0, %xmm13, %xmm13
16312        vaesenc	%xmm0, %xmm14, %xmm14
16313        vaesenc	%xmm0, %xmm15, %xmm15
16314        # aesenc_pclmul_l
16315        vpxor	%xmm2, %xmm5, %xmm5
16316        vpxor	%xmm4, %xmm6, %xmm6
16317        vpxor	%xmm3, %xmm5, %xmm5
16318        vpslldq	$8, %xmm5, %xmm1
16319        vpsrldq	$8, %xmm5, %xmm5
16320        vmovdqu	144(%rdi), %xmm4
16321        vmovdqu	L_avx2_aes_gcm_mod2_128(%rip), %xmm0
16322        vaesenc	%xmm4, %xmm8, %xmm8
16323        vpxor	%xmm1, %xmm6, %xmm6
16324        vpxor	%xmm5, %xmm7, %xmm7
16325        vpclmulqdq	$16, %xmm0, %xmm6, %xmm3
16326        vaesenc	%xmm4, %xmm9, %xmm9
16327        vaesenc	%xmm4, %xmm10, %xmm10
16328        vaesenc	%xmm4, %xmm11, %xmm11
16329        vpshufd	$0x4e, %xmm6, %xmm6
16330        vpxor	%xmm3, %xmm6, %xmm6
16331        vpclmulqdq	$16, %xmm0, %xmm6, %xmm3
16332        vaesenc	%xmm4, %xmm12, %xmm12
16333        vaesenc	%xmm4, %xmm13, %xmm13
16334        vaesenc	%xmm4, %xmm14, %xmm14
16335        vpshufd	$0x4e, %xmm6, %xmm6
16336        vpxor	%xmm3, %xmm6, %xmm6
16337        vpxor	%xmm7, %xmm6, %xmm6
16338        vaesenc	%xmm4, %xmm15, %xmm15
16339        cmpl	$11, %esi
16340        vmovdqu	160(%rdi), %xmm7
16341        jl	L_AES_GCM_decrypt_update_avx2_aesenc_128_ghash_avx_done
16342        vaesenc	%xmm7, %xmm8, %xmm8
16343        vaesenc	%xmm7, %xmm9, %xmm9
16344        vaesenc	%xmm7, %xmm10, %xmm10
16345        vaesenc	%xmm7, %xmm11, %xmm11
16346        vaesenc	%xmm7, %xmm12, %xmm12
16347        vaesenc	%xmm7, %xmm13, %xmm13
16348        vaesenc	%xmm7, %xmm14, %xmm14
16349        vaesenc	%xmm7, %xmm15, %xmm15
16350        vmovdqu	176(%rdi), %xmm7
16351        vaesenc	%xmm7, %xmm8, %xmm8
16352        vaesenc	%xmm7, %xmm9, %xmm9
16353        vaesenc	%xmm7, %xmm10, %xmm10
16354        vaesenc	%xmm7, %xmm11, %xmm11
16355        vaesenc	%xmm7, %xmm12, %xmm12
16356        vaesenc	%xmm7, %xmm13, %xmm13
16357        vaesenc	%xmm7, %xmm14, %xmm14
16358        vaesenc	%xmm7, %xmm15, %xmm15
16359        cmpl	$13, %esi
16360        vmovdqu	192(%rdi), %xmm7
16361        jl	L_AES_GCM_decrypt_update_avx2_aesenc_128_ghash_avx_done
16362        vaesenc	%xmm7, %xmm8, %xmm8
16363        vaesenc	%xmm7, %xmm9, %xmm9
16364        vaesenc	%xmm7, %xmm10, %xmm10
16365        vaesenc	%xmm7, %xmm11, %xmm11
16366        vaesenc	%xmm7, %xmm12, %xmm12
16367        vaesenc	%xmm7, %xmm13, %xmm13
16368        vaesenc	%xmm7, %xmm14, %xmm14
16369        vaesenc	%xmm7, %xmm15, %xmm15
16370        vmovdqu	208(%rdi), %xmm7
16371        vaesenc	%xmm7, %xmm8, %xmm8
16372        vaesenc	%xmm7, %xmm9, %xmm9
16373        vaesenc	%xmm7, %xmm10, %xmm10
16374        vaesenc	%xmm7, %xmm11, %xmm11
16375        vaesenc	%xmm7, %xmm12, %xmm12
16376        vaesenc	%xmm7, %xmm13, %xmm13
16377        vaesenc	%xmm7, %xmm14, %xmm14
16378        vaesenc	%xmm7, %xmm15, %xmm15
16379        vmovdqu	224(%rdi), %xmm7
16380L_AES_GCM_decrypt_update_avx2_aesenc_128_ghash_avx_done:
16381        # aesenc_last
16382        vaesenclast	%xmm7, %xmm8, %xmm8
16383        vaesenclast	%xmm7, %xmm9, %xmm9
16384        vaesenclast	%xmm7, %xmm10, %xmm10
16385        vaesenclast	%xmm7, %xmm11, %xmm11
16386        vmovdqu	(%rcx), %xmm0
16387        vmovdqu	16(%rcx), %xmm1
16388        vmovdqu	32(%rcx), %xmm2
16389        vmovdqu	48(%rcx), %xmm3
16390        vpxor	%xmm0, %xmm8, %xmm8
16391        vpxor	%xmm1, %xmm9, %xmm9
16392        vpxor	%xmm2, %xmm10, %xmm10
16393        vpxor	%xmm3, %xmm11, %xmm11
16394        vmovdqu	%xmm8, (%rdx)
16395        vmovdqu	%xmm9, 16(%rdx)
16396        vmovdqu	%xmm10, 32(%rdx)
16397        vmovdqu	%xmm11, 48(%rdx)
16398        vaesenclast	%xmm7, %xmm12, %xmm12
16399        vaesenclast	%xmm7, %xmm13, %xmm13
16400        vaesenclast	%xmm7, %xmm14, %xmm14
16401        vaesenclast	%xmm7, %xmm15, %xmm15
16402        vmovdqu	64(%rcx), %xmm0
16403        vmovdqu	80(%rcx), %xmm1
16404        vmovdqu	96(%rcx), %xmm2
16405        vmovdqu	112(%rcx), %xmm3
16406        vpxor	%xmm0, %xmm12, %xmm12
16407        vpxor	%xmm1, %xmm13, %xmm13
16408        vpxor	%xmm2, %xmm14, %xmm14
16409        vpxor	%xmm3, %xmm15, %xmm15
16410        vmovdqu	%xmm12, 64(%rdx)
16411        vmovdqu	%xmm13, 80(%rdx)
16412        vmovdqu	%xmm14, 96(%rdx)
16413        vmovdqu	%xmm15, 112(%rdx)
16414        # aesenc_128_ghash - end
16415        addl	$0x80, %r14d
16416        cmpl	%r13d, %r14d
16417        jl	L_AES_GCM_decrypt_update_avx2_ghash_128
16418        vmovdqu	(%rsp), %xmm5
16419        vmovdqu	128(%rsp), %xmm4
16420        vmovdqu	144(%rsp), %xmm15
16421L_AES_GCM_decrypt_update_avx2_done_128:
16422        cmpl	%r8d, %r14d
16423        jge	L_AES_GCM_decrypt_update_avx2_done_dec
16424        movl	%r8d, %r13d
16425        andl	$0xfffffff0, %r13d
16426        cmpl	%r13d, %r14d
16427        jge	L_AES_GCM_decrypt_update_avx2_last_block_done
16428L_AES_GCM_decrypt_update_avx2_last_block_start:
16429        vmovdqu	(%r11,%r14,1), %xmm11
16430        vpshufb	L_avx2_aes_gcm_bswap_epi64(%rip), %xmm4, %xmm10
16431        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm11, %xmm12
16432        vpaddd	L_avx2_aes_gcm_one(%rip), %xmm4, %xmm4
16433        vpxor	%xmm6, %xmm12, %xmm12
16434        # aesenc_gfmul_sb
16435        vpclmulqdq	$0x01, %xmm5, %xmm12, %xmm2
16436        vpclmulqdq	$16, %xmm5, %xmm12, %xmm3
16437        vpclmulqdq	$0x00, %xmm5, %xmm12, %xmm1
16438        vpclmulqdq	$0x11, %xmm5, %xmm12, %xmm8
16439        vpxor	(%rdi), %xmm10, %xmm10
16440        vaesenc	16(%rdi), %xmm10, %xmm10
16441        vpxor	%xmm2, %xmm3, %xmm3
16442        vpslldq	$8, %xmm3, %xmm2
16443        vpsrldq	$8, %xmm3, %xmm3
16444        vaesenc	32(%rdi), %xmm10, %xmm10
16445        vpxor	%xmm1, %xmm2, %xmm2
16446        vpclmulqdq	$16, L_avx2_aes_gcm_mod2_128(%rip), %xmm2, %xmm1
16447        vaesenc	48(%rdi), %xmm10, %xmm10
16448        vaesenc	64(%rdi), %xmm10, %xmm10
16449        vaesenc	80(%rdi), %xmm10, %xmm10
16450        vpshufd	$0x4e, %xmm2, %xmm2
16451        vpxor	%xmm1, %xmm2, %xmm2
16452        vpclmulqdq	$16, L_avx2_aes_gcm_mod2_128(%rip), %xmm2, %xmm1
16453        vaesenc	96(%rdi), %xmm10, %xmm10
16454        vaesenc	112(%rdi), %xmm10, %xmm10
16455        vaesenc	128(%rdi), %xmm10, %xmm10
16456        vpshufd	$0x4e, %xmm2, %xmm2
16457        vaesenc	144(%rdi), %xmm10, %xmm10
16458        vpxor	%xmm3, %xmm8, %xmm8
16459        vpxor	%xmm8, %xmm2, %xmm2
16460        vmovdqu	160(%rdi), %xmm0
16461        cmpl	$11, %esi
16462        jl	L_AES_GCM_decrypt_update_avx2_aesenc_gfmul_sb_last
16463        vaesenc	%xmm0, %xmm10, %xmm10
16464        vaesenc	176(%rdi), %xmm10, %xmm10
16465        vmovdqu	192(%rdi), %xmm0
16466        cmpl	$13, %esi
16467        jl	L_AES_GCM_decrypt_update_avx2_aesenc_gfmul_sb_last
16468        vaesenc	%xmm0, %xmm10, %xmm10
16469        vaesenc	208(%rdi), %xmm10, %xmm10
16470        vmovdqu	224(%rdi), %xmm0
16471L_AES_GCM_decrypt_update_avx2_aesenc_gfmul_sb_last:
16472        vaesenclast	%xmm0, %xmm10, %xmm10
16473        vpxor	%xmm1, %xmm2, %xmm6
16474        vpxor	%xmm11, %xmm10, %xmm10
16475        vmovdqu	%xmm10, (%r10,%r14,1)
16476        addl	$16, %r14d
16477        cmpl	%r13d, %r14d
16478        jl	L_AES_GCM_decrypt_update_avx2_last_block_start
16479L_AES_GCM_decrypt_update_avx2_last_block_done:
16480L_AES_GCM_decrypt_update_avx2_done_dec:
16481        vmovdqu	%xmm6, (%r9)
16482        vmovdqu	%xmm4, (%r12)
16483        vzeroupper
16484        addq	$0xa8, %rsp
16485        popq	%r14
16486        popq	%r12
16487        popq	%r13
16488        repz retq
16489#ifndef __APPLE__
16490.size	AES_GCM_decrypt_update_avx2,.-AES_GCM_decrypt_update_avx2
16491#endif /* __APPLE__ */
16492#ifndef __APPLE__
16493.text
16494.globl	AES_GCM_decrypt_final_avx2
16495.type	AES_GCM_decrypt_final_avx2,@function
16496.align	16
16497AES_GCM_decrypt_final_avx2:
16498#else
16499.section	__TEXT,__text
16500.globl	_AES_GCM_decrypt_final_avx2
16501.p2align	4
16502_AES_GCM_decrypt_final_avx2:
16503#endif /* __APPLE__ */
16504        pushq	%r12
16505        movq	16(%rsp), %rax
16506        movq	24(%rsp), %r10
16507        subq	$16, %rsp
16508        vmovdqu	(%rdi), %xmm4
16509        vmovdqu	(%r9), %xmm5
16510        vmovdqu	(%rax), %xmm6
16511        vpsrlq	$63, %xmm5, %xmm1
16512        vpsllq	$0x01, %xmm5, %xmm0
16513        vpslldq	$8, %xmm1, %xmm1
16514        vpor	%xmm1, %xmm0, %xmm0
16515        vpshufd	$0xff, %xmm5, %xmm5
16516        vpsrad	$31, %xmm5, %xmm5
16517        vpand	L_avx2_aes_gcm_mod2_128(%rip), %xmm5, %xmm5
16518        vpxor	%xmm0, %xmm5, %xmm5
16519        # calc_tag
16520        shlq	$3, %rcx
16521        shlq	$3, %r8
16522        vmovq	%rcx, %xmm0
16523        vmovq	%r8, %xmm1
16524        vpunpcklqdq	%xmm1, %xmm0, %xmm0
16525        vpxor	%xmm4, %xmm0, %xmm0
16526        # ghash_gfmul_red
16527        vpclmulqdq	$16, %xmm5, %xmm0, %xmm7
16528        vpclmulqdq	$0x01, %xmm5, %xmm0, %xmm3
16529        vpclmulqdq	$0x00, %xmm5, %xmm0, %xmm2
16530        vpxor	%xmm3, %xmm7, %xmm7
16531        vpslldq	$8, %xmm7, %xmm3
16532        vpsrldq	$8, %xmm7, %xmm7
16533        vpxor	%xmm2, %xmm3, %xmm3
16534        vpclmulqdq	$0x11, %xmm5, %xmm0, %xmm0
16535        vpclmulqdq	$16, L_avx2_aes_gcm_mod2_128(%rip), %xmm3, %xmm2
16536        vpshufd	$0x4e, %xmm3, %xmm3
16537        vpxor	%xmm2, %xmm3, %xmm3
16538        vpclmulqdq	$16, L_avx2_aes_gcm_mod2_128(%rip), %xmm3, %xmm2
16539        vpshufd	$0x4e, %xmm3, %xmm3
16540        vpxor	%xmm7, %xmm0, %xmm0
16541        vpxor	%xmm3, %xmm0, %xmm0
16542        vpxor	%xmm2, %xmm0, %xmm0
16543        vpshufb	L_avx2_aes_gcm_bswap_mask(%rip), %xmm0, %xmm0
16544        vpxor	%xmm6, %xmm0, %xmm0
16545        # cmp_tag
16546        cmpl	$16, %edx
16547        je	L_AES_GCM_decrypt_final_avx2_cmp_tag_16
16548        xorq	%r11, %r11
16549        xorq	%r9, %r9
16550        vmovdqu	%xmm0, (%rsp)
16551L_AES_GCM_decrypt_final_avx2_cmp_tag_loop:
16552        movzbl	(%rsp,%r11,1), %r12d
16553        xorb	(%rsi,%r11,1), %r12b
16554        orb	%r12b, %r9b
16555        incl	%r11d
16556        cmpl	%edx, %r11d
16557        jne	L_AES_GCM_decrypt_final_avx2_cmp_tag_loop
16558        cmpb	$0x00, %r9b
16559        sete	%r9b
16560        jmp	L_AES_GCM_decrypt_final_avx2_cmp_tag_done
16561L_AES_GCM_decrypt_final_avx2_cmp_tag_16:
16562        vmovdqu	(%rsi), %xmm1
16563        vpcmpeqb	%xmm1, %xmm0, %xmm0
16564        vpmovmskb	%xmm0, %r11
16565        # %%edx == 0xFFFF then return 1 else => return 0
16566        xorl	%r9d, %r9d
16567        cmpl	$0xffff, %r11d
16568        sete	%r9b
16569L_AES_GCM_decrypt_final_avx2_cmp_tag_done:
16570        movl	%r9d, (%r10)
16571        vzeroupper
16572        addq	$16, %rsp
16573        popq	%r12
16574        repz retq
16575#ifndef __APPLE__
16576.size	AES_GCM_decrypt_final_avx2,.-AES_GCM_decrypt_final_avx2
16577#endif /* __APPLE__ */
16578#endif /* WOLFSSL_AESGCM_STREAM */
16579#endif /* HAVE_INTEL_AVX2 */
16580#endif /* WOLFSSL_X86_64_BUILD */
16581
16582#if defined(__linux__) && defined(__ELF__)
16583.section	.note.GNU-stack,"",%progbits
16584#endif