cjson
.github
workflows CI.yml ci-fuzz.yml
CONTRIBUTING.md
fuzzing
inputs test1 test10 test11 test2 test3 test3.bu test3.uf test3.uu test4 test5 test6 test7 test8 test9
.gitignore CMakeLists.txt afl-prepare-linux.sh afl.c afl.sh cjson_read_fuzzer.c fuzz_main.c json.dict ossfuzz.sh
library_config cJSONConfig.cmake.in cJSONConfigVersion.cmake.in libcjson.pc.in libcjson_utils.pc.in uninstall.cmake
tests
inputs test1 test1.expected test10 test10.expected test11 test11.expected test2 test2.expected test3 test3.expected test4 test4.expected test5 test5.expected test6 test7 test7.expected test8 test8.expected test9 test9.expected
json-patch-tests .editorconfig .gitignore .npmignore README.md cjson-utils-tests.json package.json spec_tests.json tests.json
unity
auto colour_prompt.rb colour_reporter.rb generate_config.yml generate_module.rb generate_test_runner.rb parse_output.rb stylize_as_junit.rb test_file_filter.rb type_sanitizer.rb unity_test_summary.py unity_test_summary.rb unity_to_junit.py
docs ThrowTheSwitchCodingStandard.md UnityAssertionsCheatSheetSuitableforPrintingandPossiblyFraming.pdf UnityAssertionsReference.md UnityConfigurationGuide.md UnityGettingStartedGuide.md UnityHelperScriptsGuide.md license.txt
examples
example_1
src ProductionCode.c ProductionCode.h ProductionCode2.c ProductionCode2.h
makefile readme.txt
example_2
src ProductionCode.c ProductionCode.h ProductionCode2.c ProductionCode2.h
makefile readme.txt
example_3
helper UnityHelper.c UnityHelper.h
src ProductionCode.c ProductionCode.h ProductionCode2.c ProductionCode2.h
rakefile.rb rakefile_helper.rb readme.txt target_gcc_32.yml
unity_config.h
extras
eclipse error_parsers.txt
fixture
src unity_fixture.c unity_fixture.h unity_fixture_internals.h unity_fixture_malloc_overrides.h
rakefile.rb rakefile_helper.rb readme.txt
release build.info version.info
src unity.c unity.h unity_internals.h
.gitattributes .gitignore .travis.yml README.md
CMakeLists.txt cjson_add.c common.h compare_tests.c json_patch_tests.c minify_tests.c misc_tests.c misc_utils_tests.c old_utils_tests.c parse_array.c parse_examples.c parse_hex4.c parse_number.c parse_object.c parse_string.c parse_value.c parse_with_opts.c print_array.c print_number.c print_object.c print_string.c print_value.c readme_examples.c unity_setup.c
.editorconfig .gitattributes .gitignore .travis.yml CHANGELOG.md CMakeLists.txt CONTRIBUTORS.md LICENSE Makefile README.md SECURITY.md appveyor.yml cJSON.c cJSON.h cJSON_Utils.c cJSON_Utils.h test.c valgrind.supp
curl
.circleci config.yml
.github
ISSUE_TEMPLATE bug_report.yml config.yml docs.yml
scripts cleancmd.pl cmp-config.pl cmp-pkg-config.sh codespell-ignore.words codespell.sh distfiles.sh pyspelling.words pyspelling.yaml randcurl.pl requirements-docs.txt requirements-proselint.txt requirements.txt shellcheck-ci.sh shellcheck.sh spellcheck.curl trimmarkdownheader.pl typos.sh typos.toml verify-examples.pl verify-synopsis.pl yamlcheck.sh yamlcheck.yaml
workflows appveyor-status.yml checkdocs.yml checksrc.yml checkurls.yml codeql.yml configure-vs-cmake.yml curl-for-win.yml distcheck.yml fuzz.yml http3-linux.yml label.yml linux-old.yml linux.yml macos.yml non-native.yml windows.yml
CODEOWNERS CONTRIBUTING.md FUNDING.yml dependabot.yml labeler.yml lock.yml stale.yml
CMake CurlSymbolHiding.cmake CurlTests.c FindBrotli.cmake FindCares.cmake FindGSS.cmake FindGnuTLS.cmake FindLDAP.cmake FindLibbacktrace.cmake FindLibgsasl.cmake FindLibidn2.cmake FindLibpsl.cmake FindLibssh.cmake FindLibssh2.cmake FindLibuv.cmake FindMbedTLS.cmake FindNGHTTP2.cmake FindNGHTTP3.cmake FindNGTCP2.cmake FindNettle.cmake FindQuiche.cmake FindRustls.cmake FindWolfSSL.cmake FindZstd.cmake Macros.cmake OtherTests.cmake PickyWarnings.cmake Utilities.cmake cmake_uninstall.in.cmake curl-config.in.cmake unix-cache.cmake win32-cache.cmake
LICENSES BSD-4-Clause-UC.txt ISC.txt curl.txt
docs
cmdline-opts .gitignore CMakeLists.txt MANPAGE.md Makefile.am Makefile.inc _AUTHORS.md _BUGS.md _DESCRIPTION.md _ENVIRONMENT.md _EXITCODES.md _FILES.md _GLOBBING.md _NAME.md _OPTIONS.md _OUTPUT.md _PROGRESS.md _PROTOCOLS.md _PROXYPREFIX.md _SEEALSO.md _SYNOPSIS.md _URL.md _VARIABLES.md _VERSION.md _WWW.md abstract-unix-socket.md alt-svc.md anyauth.md append.md aws-sigv4.md basic.md ca-native.md cacert.md capath.md cert-status.md cert-type.md cert.md ciphers.md compressed-ssh.md compressed.md config.md connect-timeout.md connect-to.md continue-at.md cookie-jar.md cookie.md create-dirs.md create-file-mode.md crlf.md crlfile.md curves.md data-ascii.md data-binary.md data-raw.md data-urlencode.md data.md delegation.md digest.md disable-eprt.md disable-epsv.md disable.md disallow-username-in-url.md dns-interface.md dns-ipv4-addr.md dns-ipv6-addr.md dns-servers.md doh-cert-status.md doh-insecure.md doh-url.md dump-ca-embed.md dump-header.md ech.md egd-file.md engine.md etag-compare.md etag-save.md expect100-timeout.md fail-early.md fail-with-body.md fail.md false-start.md follow.md form-escape.md form-string.md form.md ftp-account.md ftp-alternative-to-user.md ftp-create-dirs.md ftp-method.md ftp-pasv.md ftp-port.md ftp-pret.md ftp-skip-pasv-ip.md ftp-ssl-ccc-mode.md ftp-ssl-ccc.md ftp-ssl-control.md get.md globoff.md happy-eyeballs-timeout-ms.md haproxy-clientip.md haproxy-protocol.md head.md header.md help.md hostpubmd5.md hostpubsha256.md hsts.md http0.9.md http1.0.md http1.1.md http2-prior-knowledge.md http2.md http3-only.md http3.md ignore-content-length.md insecure.md interface.md ip-tos.md ipfs-gateway.md ipv4.md ipv6.md json.md junk-session-cookies.md keepalive-cnt.md keepalive-time.md key-type.md key.md knownhosts.md krb.md libcurl.md limit-rate.md list-only.md local-port.md location-trusted.md location.md login-options.md mail-auth.md mail-from.md mail-rcpt-allowfails.md mail-rcpt.md mainpage.idx manual.md max-filesize.md max-redirs.md max-time.md metalink.md mptcp.md negotiate.md netrc-file.md netrc-optional.md netrc.md next.md no-alpn.md no-buffer.md no-clobber.md no-keepalive.md no-npn.md no-progress-meter.md no-sessionid.md noproxy.md ntlm-wb.md ntlm.md oauth2-bearer.md out-null.md output-dir.md output.md parallel-immediate.md parallel-max-host.md parallel-max.md parallel.md pass.md path-as-is.md pinnedpubkey.md post301.md post302.md post303.md preproxy.md progress-bar.md proto-default.md proto-redir.md proto.md proxy-anyauth.md proxy-basic.md proxy-ca-native.md proxy-cacert.md proxy-capath.md proxy-cert-type.md proxy-cert.md proxy-ciphers.md proxy-crlfile.md proxy-digest.md proxy-header.md proxy-http2.md proxy-insecure.md proxy-key-type.md proxy-key.md proxy-negotiate.md proxy-ntlm.md proxy-pass.md proxy-pinnedpubkey.md proxy-service-name.md proxy-ssl-allow-beast.md proxy-ssl-auto-client-cert.md proxy-tls13-ciphers.md proxy-tlsauthtype.md proxy-tlspassword.md proxy-tlsuser.md proxy-tlsv1.md proxy-user.md proxy.md proxy1.0.md proxytunnel.md pubkey.md quote.md random-file.md range.md rate.md raw.md referer.md remote-header-name.md remote-name-all.md remote-name.md remote-time.md remove-on-error.md request-target.md request.md resolve.md retry-all-errors.md retry-connrefused.md retry-delay.md retry-max-time.md retry.md sasl-authzid.md sasl-ir.md service-name.md show-error.md show-headers.md sigalgs.md silent.md skip-existing.md socks4.md socks4a.md socks5-basic.md socks5-gssapi-nec.md socks5-gssapi-service.md socks5-gssapi.md socks5-hostname.md socks5.md speed-limit.md speed-time.md ssl-allow-beast.md ssl-auto-client-cert.md ssl-no-revoke.md ssl-reqd.md ssl-revoke-best-effort.md ssl-sessions.md ssl.md sslv2.md sslv3.md stderr.md styled-output.md suppress-connect-headers.md tcp-fastopen.md tcp-nodelay.md telnet-option.md tftp-blksize.md tftp-no-options.md time-cond.md tls-earlydata.md tls-max.md tls13-ciphers.md tlsauthtype.md tlspassword.md tlsuser.md tlsv1.0.md tlsv1.1.md tlsv1.2.md tlsv1.3.md tlsv1.md tr-encoding.md trace-ascii.md trace-config.md trace-ids.md trace-time.md trace.md unix-socket.md upload-file.md upload-flags.md url-query.md url.md use-ascii.md user-agent.md user.md variable.md verbose.md version.md vlan-priority.md write-out.md xattr.md
examples .checksrc .gitignore 10-at-a-time.c CMakeLists.txt Makefile.am Makefile.example Makefile.inc README.md adddocsref.pl address-scope.c altsvc.c anyauthput.c block_ip.c cacertinmem.c certinfo.c chkspeed.c connect-to.c cookie_interface.c crawler.c debug.c default-scheme.c ephiperfifo.c evhiperfifo.c externalsocket.c fileupload.c ftp-delete.c ftp-wildcard.c ftpget.c ftpgetinfo.c ftpgetresp.c ftpsget.c ftpupload.c ftpuploadfrommem.c ftpuploadresume.c getinfo.c getinmemory.c getredirect.c getreferrer.c ghiper.c headerapi.c hiperfifo.c hsts-preload.c htmltidy.c htmltitle.cpp http-options.c http-post.c http2-download.c http2-pushinmemory.c http2-serverpush.c http2-upload.c http3-present.c http3.c httpcustomheader.c httpput-postfields.c httpput.c https.c imap-append.c imap-authzid.c imap-copy.c imap-create.c imap-delete.c imap-examine.c imap-fetch.c imap-list.c imap-lsub.c imap-multi.c imap-noop.c imap-search.c imap-ssl.c imap-store.c imap-tls.c interface.c ipv6.c keepalive.c localport.c log_failed_transfers.c maxconnects.c multi-app.c multi-debugcallback.c multi-double.c multi-event.c multi-formadd.c multi-legacy.c multi-post.c multi-single.c multi-uv.c netrc.c parseurl.c persistent.c pop3-authzid.c pop3-dele.c pop3-list.c pop3-multi.c pop3-noop.c pop3-retr.c pop3-ssl.c pop3-stat.c pop3-tls.c pop3-top.c pop3-uidl.c post-callback.c postinmemory.c postit2-formadd.c postit2.c progressfunc.c protofeats.c range.c resolve.c rtsp-options.c sendrecv.c sepheaders.c sessioninfo.c sftpget.c sftpuploadresume.c shared-connection-cache.c simple.c simplepost.c simplessl.c smooth-gtk-thread.c smtp-authzid.c smtp-expn.c smtp-mail.c smtp-mime.c smtp-multi.c smtp-ssl.c smtp-tls.c smtp-vrfy.c sslbackend.c synctime.c threaded.c unixsocket.c url2file.c urlapi.c usercertinmem.c version-check.pl websocket-cb.c websocket-updown.c websocket.c xmlstream.c
internals BUFQ.md BUFREF.md CHECKSRC.md CLIENT-READERS.md CLIENT-WRITERS.md CODE_STYLE.md CONNECTION-FILTERS.md CREDENTIALS.md CURLX.md DYNBUF.md HASH.md LLIST.md MID.md MQTT.md MULTI-EV.md NEW-PROTOCOL.md PEERS.md PORTING.md RATELIMITS.md README.md SCORECARD.md SPLAY.md STRPARSE.md THRDPOOL-AND-QUEUE.md TIME-KEEPING.md TLS-SESSIONS.md UINT_SETS.md WEBSOCKET.md
libcurl
opts CMakeLists.txt CURLINFO_ACTIVESOCKET.md CURLINFO_APPCONNECT_TIME.md CURLINFO_APPCONNECT_TIME_T.md CURLINFO_CAINFO.md CURLINFO_CAPATH.md CURLINFO_CERTINFO.md CURLINFO_CONDITION_UNMET.md CURLINFO_CONNECT_TIME.md CURLINFO_CONNECT_TIME_T.md CURLINFO_CONN_ID.md CURLINFO_CONTENT_LENGTH_DOWNLOAD.md CURLINFO_CONTENT_LENGTH_DOWNLOAD_T.md CURLINFO_CONTENT_LENGTH_UPLOAD.md CURLINFO_CONTENT_LENGTH_UPLOAD_T.md CURLINFO_CONTENT_TYPE.md CURLINFO_COOKIELIST.md CURLINFO_EARLYDATA_SENT_T.md CURLINFO_EFFECTIVE_METHOD.md CURLINFO_EFFECTIVE_URL.md CURLINFO_FILETIME.md CURLINFO_FILETIME_T.md CURLINFO_FTP_ENTRY_PATH.md CURLINFO_HEADER_SIZE.md CURLINFO_HTTPAUTH_AVAIL.md CURLINFO_HTTPAUTH_USED.md CURLINFO_HTTP_CONNECTCODE.md CURLINFO_HTTP_VERSION.md CURLINFO_LASTSOCKET.md CURLINFO_LOCAL_IP.md CURLINFO_LOCAL_PORT.md CURLINFO_NAMELOOKUP_TIME.md CURLINFO_NAMELOOKUP_TIME_T.md CURLINFO_NUM_CONNECTS.md CURLINFO_OS_ERRNO.md CURLINFO_POSTTRANSFER_TIME_T.md CURLINFO_PRETRANSFER_TIME.md CURLINFO_PRETRANSFER_TIME_T.md CURLINFO_PRIMARY_IP.md CURLINFO_PRIMARY_PORT.md CURLINFO_PRIVATE.md CURLINFO_PROTOCOL.md CURLINFO_PROXYAUTH_AVAIL.md CURLINFO_PROXYAUTH_USED.md CURLINFO_PROXY_ERROR.md CURLINFO_PROXY_SSL_VERIFYRESULT.md CURLINFO_QUEUE_TIME_T.md CURLINFO_REDIRECT_COUNT.md CURLINFO_REDIRECT_TIME.md CURLINFO_REDIRECT_TIME_T.md CURLINFO_REDIRECT_URL.md CURLINFO_REFERER.md CURLINFO_REQUEST_SIZE.md CURLINFO_RESPONSE_CODE.md CURLINFO_RETRY_AFTER.md CURLINFO_RTSP_CLIENT_CSEQ.md CURLINFO_RTSP_CSEQ_RECV.md CURLINFO_RTSP_SERVER_CSEQ.md CURLINFO_RTSP_SESSION_ID.md CURLINFO_SCHEME.md CURLINFO_SIZE_DELIVERED.md CURLINFO_SIZE_DOWNLOAD.md CURLINFO_SIZE_DOWNLOAD_T.md CURLINFO_SIZE_UPLOAD.md CURLINFO_SIZE_UPLOAD_T.md CURLINFO_SPEED_DOWNLOAD.md CURLINFO_SPEED_DOWNLOAD_T.md CURLINFO_SPEED_UPLOAD.md CURLINFO_SPEED_UPLOAD_T.md CURLINFO_SSL_ENGINES.md CURLINFO_SSL_VERIFYRESULT.md CURLINFO_STARTTRANSFER_TIME.md CURLINFO_STARTTRANSFER_TIME_T.md CURLINFO_TLS_SESSION.md CURLINFO_TLS_SSL_PTR.md CURLINFO_TOTAL_TIME.md CURLINFO_TOTAL_TIME_T.md CURLINFO_USED_PROXY.md CURLINFO_XFER_ID.md CURLMINFO_XFERS_ADDED.md CURLMINFO_XFERS_CURRENT.md CURLMINFO_XFERS_DONE.md CURLMINFO_XFERS_PENDING.md CURLMINFO_XFERS_RUNNING.md CURLMOPT_CHUNK_LENGTH_PENALTY_SIZE.md CURLMOPT_CONTENT_LENGTH_PENALTY_SIZE.md CURLMOPT_MAXCONNECTS.md CURLMOPT_MAX_CONCURRENT_STREAMS.md CURLMOPT_MAX_HOST_CONNECTIONS.md CURLMOPT_MAX_PIPELINE_LENGTH.md CURLMOPT_MAX_TOTAL_CONNECTIONS.md CURLMOPT_NETWORK_CHANGED.md CURLMOPT_NOTIFYDATA.md CURLMOPT_NOTIFYFUNCTION.md CURLMOPT_PIPELINING.md CURLMOPT_PIPELINING_SERVER_BL.md CURLMOPT_PIPELINING_SITE_BL.md CURLMOPT_PUSHDATA.md CURLMOPT_PUSHFUNCTION.md CURLMOPT_QUICK_EXIT.md CURLMOPT_RESOLVE_THREADS_MAX.md CURLMOPT_SOCKETDATA.md CURLMOPT_SOCKETFUNCTION.md CURLMOPT_TIMERDATA.md CURLMOPT_TIMERFUNCTION.md CURLOPT_ABSTRACT_UNIX_SOCKET.md CURLOPT_ACCEPTTIMEOUT_MS.md CURLOPT_ACCEPT_ENCODING.md CURLOPT_ADDRESS_SCOPE.md CURLOPT_ALTSVC.md CURLOPT_ALTSVC_CTRL.md CURLOPT_APPEND.md CURLOPT_AUTOREFERER.md CURLOPT_AWS_SIGV4.md CURLOPT_BUFFERSIZE.md CURLOPT_CAINFO.md CURLOPT_CAINFO_BLOB.md CURLOPT_CAPATH.md CURLOPT_CA_CACHE_TIMEOUT.md CURLOPT_CERTINFO.md CURLOPT_CHUNK_BGN_FUNCTION.md CURLOPT_CHUNK_DATA.md CURLOPT_CHUNK_END_FUNCTION.md CURLOPT_CLOSESOCKETDATA.md CURLOPT_CLOSESOCKETFUNCTION.md CURLOPT_CONNECTTIMEOUT.md CURLOPT_CONNECTTIMEOUT_MS.md CURLOPT_CONNECT_ONLY.md CURLOPT_CONNECT_TO.md CURLOPT_CONV_FROM_NETWORK_FUNCTION.md CURLOPT_CONV_FROM_UTF8_FUNCTION.md CURLOPT_CONV_TO_NETWORK_FUNCTION.md CURLOPT_COOKIE.md CURLOPT_COOKIEFILE.md CURLOPT_COOKIEJAR.md CURLOPT_COOKIELIST.md CURLOPT_COOKIESESSION.md CURLOPT_COPYPOSTFIELDS.md CURLOPT_CRLF.md CURLOPT_CRLFILE.md CURLOPT_CURLU.md CURLOPT_CUSTOMREQUEST.md CURLOPT_DEBUGDATA.md CURLOPT_DEBUGFUNCTION.md CURLOPT_DEFAULT_PROTOCOL.md CURLOPT_DIRLISTONLY.md CURLOPT_DISALLOW_USERNAME_IN_URL.md CURLOPT_DNS_CACHE_TIMEOUT.md CURLOPT_DNS_INTERFACE.md CURLOPT_DNS_LOCAL_IP4.md CURLOPT_DNS_LOCAL_IP6.md CURLOPT_DNS_SERVERS.md CURLOPT_DNS_SHUFFLE_ADDRESSES.md CURLOPT_DNS_USE_GLOBAL_CACHE.md CURLOPT_DOH_SSL_VERIFYHOST.md CURLOPT_DOH_SSL_VERIFYPEER.md CURLOPT_DOH_SSL_VERIFYSTATUS.md CURLOPT_DOH_URL.md CURLOPT_ECH.md CURLOPT_EGDSOCKET.md CURLOPT_ERRORBUFFER.md CURLOPT_EXPECT_100_TIMEOUT_MS.md CURLOPT_FAILONERROR.md CURLOPT_FILETIME.md CURLOPT_FNMATCH_DATA.md CURLOPT_FNMATCH_FUNCTION.md CURLOPT_FOLLOWLOCATION.md CURLOPT_FORBID_REUSE.md CURLOPT_FRESH_CONNECT.md CURLOPT_FTPPORT.md CURLOPT_FTPSSLAUTH.md CURLOPT_FTP_ACCOUNT.md CURLOPT_FTP_ALTERNATIVE_TO_USER.md CURLOPT_FTP_CREATE_MISSING_DIRS.md CURLOPT_FTP_FILEMETHOD.md CURLOPT_FTP_SKIP_PASV_IP.md CURLOPT_FTP_SSL_CCC.md CURLOPT_FTP_USE_EPRT.md CURLOPT_FTP_USE_EPSV.md CURLOPT_FTP_USE_PRET.md CURLOPT_GSSAPI_DELEGATION.md CURLOPT_HAPPY_EYEBALLS_TIMEOUT_MS.md CURLOPT_HAPROXYPROTOCOL.md CURLOPT_HAPROXY_CLIENT_IP.md CURLOPT_HEADER.md CURLOPT_HEADERDATA.md CURLOPT_HEADERFUNCTION.md CURLOPT_HEADEROPT.md CURLOPT_HSTS.md CURLOPT_HSTSREADDATA.md CURLOPT_HSTSREADFUNCTION.md CURLOPT_HSTSWRITEDATA.md CURLOPT_HSTSWRITEFUNCTION.md CURLOPT_HSTS_CTRL.md CURLOPT_HTTP09_ALLOWED.md CURLOPT_HTTP200ALIASES.md CURLOPT_HTTPAUTH.md CURLOPT_HTTPGET.md CURLOPT_HTTPHEADER.md CURLOPT_HTTPPOST.md CURLOPT_HTTPPROXYTUNNEL.md CURLOPT_HTTP_CONTENT_DECODING.md CURLOPT_HTTP_TRANSFER_DECODING.md CURLOPT_HTTP_VERSION.md CURLOPT_IGNORE_CONTENT_LENGTH.md CURLOPT_INFILESIZE.md CURLOPT_INFILESIZE_LARGE.md CURLOPT_INTERFACE.md CURLOPT_INTERLEAVEDATA.md CURLOPT_INTERLEAVEFUNCTION.md CURLOPT_IOCTLDATA.md CURLOPT_IOCTLFUNCTION.md CURLOPT_IPRESOLVE.md CURLOPT_ISSUERCERT.md CURLOPT_ISSUERCERT_BLOB.md CURLOPT_KEEP_SENDING_ON_ERROR.md CURLOPT_KEYPASSWD.md CURLOPT_KRBLEVEL.md CURLOPT_LOCALPORT.md CURLOPT_LOCALPORTRANGE.md CURLOPT_LOGIN_OPTIONS.md CURLOPT_LOW_SPEED_LIMIT.md CURLOPT_LOW_SPEED_TIME.md CURLOPT_MAIL_AUTH.md CURLOPT_MAIL_FROM.md CURLOPT_MAIL_RCPT.md CURLOPT_MAIL_RCPT_ALLOWFAILS.md CURLOPT_MAXAGE_CONN.md CURLOPT_MAXCONNECTS.md CURLOPT_MAXFILESIZE.md CURLOPT_MAXFILESIZE_LARGE.md CURLOPT_MAXLIFETIME_CONN.md CURLOPT_MAXREDIRS.md CURLOPT_MAX_RECV_SPEED_LARGE.md CURLOPT_MAX_SEND_SPEED_LARGE.md CURLOPT_MIMEPOST.md CURLOPT_MIME_OPTIONS.md CURLOPT_NETRC.md CURLOPT_NETRC_FILE.md CURLOPT_NEW_DIRECTORY_PERMS.md CURLOPT_NEW_FILE_PERMS.md CURLOPT_NOBODY.md CURLOPT_NOPROGRESS.md CURLOPT_NOPROXY.md CURLOPT_NOSIGNAL.md CURLOPT_OPENSOCKETDATA.md CURLOPT_OPENSOCKETFUNCTION.md CURLOPT_PASSWORD.md CURLOPT_PATH_AS_IS.md CURLOPT_PINNEDPUBLICKEY.md CURLOPT_PIPEWAIT.md CURLOPT_PORT.md CURLOPT_POST.md CURLOPT_POSTFIELDS.md CURLOPT_POSTFIELDSIZE.md CURLOPT_POSTFIELDSIZE_LARGE.md CURLOPT_POSTQUOTE.md CURLOPT_POSTREDIR.md CURLOPT_PREQUOTE.md CURLOPT_PREREQDATA.md CURLOPT_PREREQFUNCTION.md CURLOPT_PRE_PROXY.md CURLOPT_PRIVATE.md CURLOPT_PROGRESSDATA.md CURLOPT_PROGRESSFUNCTION.md CURLOPT_PROTOCOLS.md CURLOPT_PROTOCOLS_STR.md CURLOPT_PROXY.md CURLOPT_PROXYAUTH.md CURLOPT_PROXYHEADER.md CURLOPT_PROXYPASSWORD.md CURLOPT_PROXYPORT.md CURLOPT_PROXYTYPE.md CURLOPT_PROXYUSERNAME.md CURLOPT_PROXYUSERPWD.md CURLOPT_PROXY_CAINFO.md CURLOPT_PROXY_CAINFO_BLOB.md CURLOPT_PROXY_CAPATH.md CURLOPT_PROXY_CRLFILE.md CURLOPT_PROXY_ISSUERCERT.md CURLOPT_PROXY_ISSUERCERT_BLOB.md CURLOPT_PROXY_KEYPASSWD.md CURLOPT_PROXY_PINNEDPUBLICKEY.md CURLOPT_PROXY_SERVICE_NAME.md CURLOPT_PROXY_SSLCERT.md CURLOPT_PROXY_SSLCERTTYPE.md CURLOPT_PROXY_SSLCERT_BLOB.md CURLOPT_PROXY_SSLKEY.md CURLOPT_PROXY_SSLKEYTYPE.md CURLOPT_PROXY_SSLKEY_BLOB.md CURLOPT_PROXY_SSLVERSION.md CURLOPT_PROXY_SSL_CIPHER_LIST.md CURLOPT_PROXY_SSL_OPTIONS.md CURLOPT_PROXY_SSL_VERIFYHOST.md CURLOPT_PROXY_SSL_VERIFYPEER.md CURLOPT_PROXY_TLS13_CIPHERS.md CURLOPT_PROXY_TLSAUTH_PASSWORD.md CURLOPT_PROXY_TLSAUTH_TYPE.md CURLOPT_PROXY_TLSAUTH_USERNAME.md CURLOPT_PROXY_TRANSFER_MODE.md CURLOPT_PUT.md CURLOPT_QUICK_EXIT.md CURLOPT_QUOTE.md CURLOPT_RANDOM_FILE.md CURLOPT_RANGE.md CURLOPT_READDATA.md CURLOPT_READFUNCTION.md CURLOPT_REDIR_PROTOCOLS.md CURLOPT_REDIR_PROTOCOLS_STR.md CURLOPT_REFERER.md CURLOPT_REQUEST_TARGET.md CURLOPT_RESOLVE.md CURLOPT_RESOLVER_START_DATA.md CURLOPT_RESOLVER_START_FUNCTION.md CURLOPT_RESUME_FROM.md CURLOPT_RESUME_FROM_LARGE.md CURLOPT_RTSP_CLIENT_CSEQ.md CURLOPT_RTSP_REQUEST.md CURLOPT_RTSP_SERVER_CSEQ.md CURLOPT_RTSP_SESSION_ID.md CURLOPT_RTSP_STREAM_URI.md CURLOPT_RTSP_TRANSPORT.md CURLOPT_SASL_AUTHZID.md CURLOPT_SASL_IR.md CURLOPT_SEEKDATA.md CURLOPT_SEEKFUNCTION.md CURLOPT_SERVER_RESPONSE_TIMEOUT.md CURLOPT_SERVER_RESPONSE_TIMEOUT_MS.md CURLOPT_SERVICE_NAME.md CURLOPT_SHARE.md CURLOPT_SOCKOPTDATA.md CURLOPT_SOCKOPTFUNCTION.md CURLOPT_SOCKS5_AUTH.md CURLOPT_SOCKS5_GSSAPI_NEC.md CURLOPT_SOCKS5_GSSAPI_SERVICE.md CURLOPT_SSH_AUTH_TYPES.md CURLOPT_SSH_COMPRESSION.md CURLOPT_SSH_HOSTKEYDATA.md CURLOPT_SSH_HOSTKEYFUNCTION.md CURLOPT_SSH_HOST_PUBLIC_KEY_MD5.md CURLOPT_SSH_HOST_PUBLIC_KEY_SHA256.md CURLOPT_SSH_KEYDATA.md CURLOPT_SSH_KEYFUNCTION.md CURLOPT_SSH_KNOWNHOSTS.md CURLOPT_SSH_PRIVATE_KEYFILE.md CURLOPT_SSH_PUBLIC_KEYFILE.md CURLOPT_SSLCERT.md CURLOPT_SSLCERTTYPE.md CURLOPT_SSLCERT_BLOB.md CURLOPT_SSLENGINE.md CURLOPT_SSLENGINE_DEFAULT.md CURLOPT_SSLKEY.md CURLOPT_SSLKEYTYPE.md CURLOPT_SSLKEY_BLOB.md CURLOPT_SSLVERSION.md CURLOPT_SSL_CIPHER_LIST.md CURLOPT_SSL_CTX_DATA.md CURLOPT_SSL_CTX_FUNCTION.md CURLOPT_SSL_EC_CURVES.md CURLOPT_SSL_ENABLE_ALPN.md CURLOPT_SSL_ENABLE_NPN.md CURLOPT_SSL_FALSESTART.md CURLOPT_SSL_OPTIONS.md CURLOPT_SSL_SESSIONID_CACHE.md CURLOPT_SSL_SIGNATURE_ALGORITHMS.md CURLOPT_SSL_VERIFYHOST.md CURLOPT_SSL_VERIFYPEER.md CURLOPT_SSL_VERIFYSTATUS.md CURLOPT_STDERR.md CURLOPT_STREAM_DEPENDS.md CURLOPT_STREAM_DEPENDS_E.md CURLOPT_STREAM_WEIGHT.md CURLOPT_SUPPRESS_CONNECT_HEADERS.md CURLOPT_TCP_FASTOPEN.md CURLOPT_TCP_KEEPALIVE.md CURLOPT_TCP_KEEPCNT.md CURLOPT_TCP_KEEPIDLE.md CURLOPT_TCP_KEEPINTVL.md CURLOPT_TCP_NODELAY.md CURLOPT_TELNETOPTIONS.md CURLOPT_TFTP_BLKSIZE.md CURLOPT_TFTP_NO_OPTIONS.md CURLOPT_TIMECONDITION.md CURLOPT_TIMEOUT.md CURLOPT_TIMEOUT_MS.md CURLOPT_TIMEVALUE.md CURLOPT_TIMEVALUE_LARGE.md CURLOPT_TLS13_CIPHERS.md CURLOPT_TLSAUTH_PASSWORD.md CURLOPT_TLSAUTH_TYPE.md CURLOPT_TLSAUTH_USERNAME.md CURLOPT_TRAILERDATA.md CURLOPT_TRAILERFUNCTION.md CURLOPT_TRANSFERTEXT.md CURLOPT_TRANSFER_ENCODING.md CURLOPT_UNIX_SOCKET_PATH.md CURLOPT_UNRESTRICTED_AUTH.md CURLOPT_UPKEEP_INTERVAL_MS.md CURLOPT_UPLOAD.md CURLOPT_UPLOAD_BUFFERSIZE.md CURLOPT_UPLOAD_FLAGS.md CURLOPT_URL.md CURLOPT_USERAGENT.md CURLOPT_USERNAME.md CURLOPT_USERPWD.md CURLOPT_USE_SSL.md CURLOPT_VERBOSE.md CURLOPT_WILDCARDMATCH.md CURLOPT_WRITEDATA.md CURLOPT_WRITEFUNCTION.md CURLOPT_WS_OPTIONS.md CURLOPT_XFERINFODATA.md CURLOPT_XFERINFOFUNCTION.md CURLOPT_XOAUTH2_BEARER.md CURLSHOPT_LOCKFUNC.md CURLSHOPT_SHARE.md CURLSHOPT_UNLOCKFUNC.md CURLSHOPT_UNSHARE.md CURLSHOPT_USERDATA.md Makefile.am Makefile.inc
.gitignore ABI.md CMakeLists.txt Makefile.am Makefile.inc curl_easy_cleanup.md curl_easy_duphandle.md curl_easy_escape.md curl_easy_getinfo.md curl_easy_header.md curl_easy_init.md curl_easy_nextheader.md curl_easy_option_by_id.md curl_easy_option_by_name.md curl_easy_option_next.md curl_easy_pause.md curl_easy_perform.md curl_easy_recv.md curl_easy_reset.md curl_easy_send.md curl_easy_setopt.md curl_easy_ssls_export.md curl_easy_ssls_import.md curl_easy_strerror.md curl_easy_unescape.md curl_easy_upkeep.md curl_escape.md curl_formadd.md curl_formfree.md curl_formget.md curl_free.md curl_getdate.md curl_getenv.md curl_global_cleanup.md curl_global_init.md curl_global_init_mem.md curl_global_sslset.md curl_global_trace.md curl_mime_addpart.md curl_mime_data.md curl_mime_data_cb.md curl_mime_encoder.md curl_mime_filedata.md curl_mime_filename.md curl_mime_free.md curl_mime_headers.md curl_mime_init.md curl_mime_name.md curl_mime_subparts.md curl_mime_type.md curl_mprintf.md curl_multi_add_handle.md curl_multi_assign.md curl_multi_cleanup.md curl_multi_fdset.md curl_multi_get_handles.md curl_multi_get_offt.md curl_multi_info_read.md curl_multi_init.md curl_multi_notify_disable.md curl_multi_notify_enable.md curl_multi_perform.md curl_multi_poll.md curl_multi_remove_handle.md curl_multi_setopt.md curl_multi_socket.md curl_multi_socket_action.md curl_multi_socket_all.md curl_multi_strerror.md curl_multi_timeout.md curl_multi_wait.md curl_multi_waitfds.md curl_multi_wakeup.md curl_pushheader_byname.md curl_pushheader_bynum.md curl_share_cleanup.md curl_share_init.md curl_share_setopt.md curl_share_strerror.md curl_slist_append.md curl_slist_free_all.md curl_strequal.md curl_strnequal.md curl_unescape.md curl_url.md curl_url_cleanup.md curl_url_dup.md curl_url_get.md curl_url_set.md curl_url_strerror.md curl_version.md curl_version_info.md curl_ws_meta.md curl_ws_recv.md curl_ws_send.md curl_ws_start_frame.md libcurl-easy.md libcurl-env-dbg.md libcurl-env.md libcurl-errors.md libcurl-multi.md libcurl-security.md libcurl-share.md libcurl-thread.md libcurl-tutorial.md libcurl-url.md libcurl-ws.md libcurl.m4 libcurl.md mksymbolsmanpage.pl symbols-in-versions symbols.pl
tests CI.md FILEFORMAT.md HTTP.md TEST-SUITE.md
.gitignore ALTSVC.md BINDINGS.md BUG-BOUNTY.md BUGS.md CIPHERS-TLS12.md CIPHERS.md CMakeLists.txt CODE_OF_CONDUCT.md CODE_REVIEW.md CONTRIBUTE.md CURL-DISABLE.md CURLDOWN.md DEPRECATE.md DISTROS.md EARLY-RELEASE.md ECH.md EXPERIMENTAL.md FAQ.md FEATURES.md GOVERNANCE.md HELP-US.md HISTORY.md HSTS.md HTTP-COOKIES.md HTTP3.md HTTPSRR.md INFRASTRUCTURE.md INSTALL-CMAKE.md INSTALL.md INTERNALS.md IPFS.md KNOWN_BUGS.md KNOWN_RISKS.md MAIL-ETIQUETTE.md MANUAL.md Makefile.am README.md RELEASE-PROCEDURE.md ROADMAP.md RUSTLS.md SECURITY-ADVISORY.md SPONSORS.md SSL-PROBLEMS.md SSLCERTS.md THANKS THANKS-filter TODO.md TheArtOfHttpScripting.md URL-SYNTAX.md VERIFY.md VERSIONS.md VULN-DISCLOSURE-POLICY.md curl-config.md mk-ca-bundle.md options-in-versions runtests.md testcurl.md wcurl.md
include
curl Makefile.am curl.h curlver.h easy.h header.h mprintf.h multi.h options.h stdcheaders.h system.h typecheck-gcc.h urlapi.h websockets.h
Makefile.am README.md
lib
curlx base64.c base64.h basename.c basename.h dynbuf.c dynbuf.h fopen.c fopen.h inet_ntop.c inet_ntop.h inet_pton.c inet_pton.h multibyte.c multibyte.h nonblock.c nonblock.h snprintf.c snprintf.h strcopy.c strcopy.h strdup.c strdup.h strerr.c strerr.h strparse.c strparse.h timediff.c timediff.h timeval.c timeval.h version_win32.c version_win32.h wait.c wait.h warnless.c warnless.h winapi.c winapi.h
vauth cleartext.c cram.c digest.c digest.h digest_sspi.c gsasl.c krb5_gssapi.c krb5_sspi.c ntlm.c ntlm_sspi.c oauth2.c spnego_gssapi.c spnego_sspi.c vauth.c vauth.h
vquic curl_ngtcp2.c curl_ngtcp2.h curl_quiche.c curl_quiche.h vquic-tls.c vquic-tls.h vquic.c vquic.h vquic_int.h
vssh libssh.c libssh2.c ssh.h vssh.c vssh.h
vtls apple.c apple.h cipher_suite.c cipher_suite.h gtls.c gtls.h hostcheck.c hostcheck.h keylog.c keylog.h mbedtls.c mbedtls.h openssl.c openssl.h rustls.c rustls.h schannel.c schannel.h schannel_int.h schannel_verify.c vtls.c vtls.h vtls_int.h vtls_scache.c vtls_scache.h vtls_spack.c vtls_spack.h wolfssl.c wolfssl.h x509asn1.c x509asn1.h
.gitignore CMakeLists.txt Makefile.am Makefile.inc Makefile.soname altsvc.c altsvc.h amigaos.c amigaos.h arpa_telnet.h asyn-ares.c asyn-base.c asyn-thrdd.c asyn.h bufq.c bufq.h bufref.c bufref.h cf-dns.c cf-dns.h cf-h1-proxy.c cf-h1-proxy.h cf-h2-proxy.c cf-h2-proxy.h cf-haproxy.c cf-haproxy.h cf-https-connect.c cf-https-connect.h cf-ip-happy.c cf-ip-happy.h cf-socket.c cf-socket.h cfilters.c cfilters.h config-mac.h config-os400.h config-riscos.h config-win32.h conncache.c conncache.h connect.c connect.h content_encoding.c content_encoding.h cookie.c cookie.h creds.c creds.h cshutdn.c cshutdn.h curl_addrinfo.c curl_addrinfo.h curl_config-cmake.h.in curl_ctype.h curl_endian.c curl_endian.h curl_fnmatch.c curl_fnmatch.h curl_fopen.c curl_fopen.h curl_get_line.c curl_get_line.h curl_gethostname.c curl_gethostname.h curl_gssapi.c curl_gssapi.h curl_hmac.h curl_ldap.h curl_md4.h curl_md5.h curl_memrchr.c curl_memrchr.h curl_ntlm_core.c curl_ntlm_core.h curl_printf.h curl_range.c curl_range.h curl_sasl.c curl_sasl.h curl_setup.h curl_sha256.h curl_sha512_256.c curl_sha512_256.h curl_share.c curl_share.h curl_sspi.c curl_sspi.h curl_threads.c curl_threads.h curl_trc.c curl_trc.h cw-out.c cw-out.h cw-pause.c cw-pause.h dict.c dict.h dllmain.c dnscache.c dnscache.h doh.c doh.h dynhds.c dynhds.h easy.c easy_lock.h easygetopt.c easyif.h easyoptions.c easyoptions.h escape.c escape.h fake_addrinfo.c fake_addrinfo.h file.c file.h fileinfo.c fileinfo.h formdata.c formdata.h ftp-int.h ftp.c ftp.h ftplistparser.c ftplistparser.h functypes.h getenv.c getinfo.c getinfo.h gopher.c gopher.h hash.c hash.h headers.c headers.h hmac.c hostip.c hostip.h hostip4.c hostip6.c hsts.c hsts.h http.c http.h http1.c http1.h http2.c http2.h http_aws_sigv4.c http_aws_sigv4.h http_chunks.c http_chunks.h http_digest.c http_digest.h http_negotiate.c http_negotiate.h http_ntlm.c http_ntlm.h http_proxy.c http_proxy.h httpsrr.c httpsrr.h idn.c idn.h if2ip.c if2ip.h imap.c imap.h ldap.c libcurl.def libcurl.rc libcurl.vers.in llist.c llist.h macos.c macos.h md4.c md5.c memdebug.c mime.c mime.h mprintf.c mqtt.c mqtt.h multi.c multi_ev.c multi_ev.h multi_ntfy.c multi_ntfy.h multihandle.h multiif.h netrc.c netrc.h noproxy.c noproxy.h openldap.c optiontable.pl parsedate.c parsedate.h peer.c peer.h pingpong.c pingpong.h pop3.c pop3.h progress.c progress.h protocol.c protocol.h psl.c psl.h rand.c rand.h ratelimit.c ratelimit.h request.c request.h rtsp.c rtsp.h select.c select.h sendf.c sendf.h setopt.c setopt.h setup-os400.h setup-vms.h setup-win32.h sha256.c sigpipe.h slist.c slist.h smb.c smb.h smtp.c smtp.h sockaddr.h socketpair.c socketpair.h socks.c socks.h socks_gssapi.c socks_sspi.c splay.c splay.h strcase.c strcase.h strequal.c strerror.c strerror.h system_win32.c system_win32.h telnet.c telnet.h tftp.c tftp.h thrdpool.c thrdpool.h thrdqueue.c thrdqueue.h transfer.c transfer.h uint-bset.c uint-bset.h uint-hash.c uint-hash.h uint-spbset.c uint-spbset.h uint-table.c uint-table.h url.c url.h urlapi-int.h urlapi.c urldata.h version.c ws.c ws.h
m4 .gitignore curl-amissl.m4 curl-apple-sectrust.m4 curl-compilers.m4 curl-confopts.m4 curl-functions.m4 curl-gnutls.m4 curl-mbedtls.m4 curl-openssl.m4 curl-override.m4 curl-reentrant.m4 curl-rustls.m4 curl-schannel.m4 curl-sysconfig.m4 curl-wolfssl.m4 xc-am-iface.m4 xc-cc-check.m4 xc-lt-iface.m4 xc-val-flgs.m4 zz40-xc-ovr.m4 zz50-xc-ovr.m4
projects
OS400
rpg-examples HEADERAPI HTTPPOST INMEMORY SIMPLE1 SIMPLE2 SMTPSRCMBR
.checksrc README.OS400 ccsidcurl.c ccsidcurl.h config400.default curl.cmd curl.inc.in curlcl.c curlmain.c initscript.sh make-docs.sh make-include.sh make-lib.sh make-src.sh make-tests.sh makefile.sh os400sys.c os400sys.h
Windows
tmpl .gitattributes README.txt curl-all.sln curl.sln curl.vcxproj curl.vcxproj.filters libcurl.sln libcurl.vcxproj libcurl.vcxproj.filters
.gitignore README.md generate.bat
vms Makefile.am backup_gnv_curl_src.com build_curl-config_script.com build_gnv_curl.com build_gnv_curl_pcsi_desc.com build_gnv_curl_pcsi_text.com build_gnv_curl_release_notes.com build_libcurl_pc.com build_vms.com clean_gnv_curl.com compare_curl_source.com config_h.com curl_crtl_init.c curl_gnv_build_steps.txt curl_release_note_start.txt curl_startup.com curlmsg.h curlmsg.msg curlmsg.sdl curlmsg_vms.h generate_config_vms_h_curl.com generate_vax_transfer.com gnv_conftest.c_first gnv_curl_configure.sh gnv_libcurl_symbols.opt gnv_link_curl.com macro32_exactcase.patch make_gnv_curl_install.sh make_pcsi_curl_kit_name.com pcsi_gnv_curl_file_list.txt pcsi_product_gnv_curl.com readme report_openssl_version.c setup_gnv_curl_build.com stage_curl_install.com vms_eco_level.h
Makefile.am README.md
scripts .checksrc CMakeLists.txt Makefile.am badwords badwords-all badwords.txt cd2cd cd2nroff cdall checksrc-all.pl checksrc.pl cmakelint.sh completion.pl contributors.sh contrithanks.sh coverage.sh delta dmaketgz extract-unit-protos firefox-db2pem.sh installcheck.sh maketgz managen mdlinkcheck mk-ca-bundle.pl mk-unity.pl nroff2cd perlcheck.sh pythonlint.sh randdisable release-notes.pl release-tools.sh schemetable.c singleuse.pl spacecheck.pl top-complexity top-length verify-release wcurl
src
toolx tool_time.c tool_time.h
.checksrc .gitignore CMakeLists.txt Makefile.am Makefile.inc config2setopts.c config2setopts.h curl.rc curlinfo.c mk-file-embed.pl mkhelp.pl slist_wc.c slist_wc.h terminal.c terminal.h tool_cb_dbg.c tool_cb_dbg.h tool_cb_hdr.c tool_cb_hdr.h tool_cb_prg.c tool_cb_prg.h tool_cb_rea.c tool_cb_rea.h tool_cb_see.c tool_cb_see.h tool_cb_soc.c tool_cb_soc.h tool_cb_wrt.c tool_cb_wrt.h tool_cfgable.c tool_cfgable.h tool_dirhie.c tool_dirhie.h tool_doswin.c tool_doswin.h tool_easysrc.c tool_easysrc.h tool_filetime.c tool_filetime.h tool_findfile.c tool_findfile.h tool_formparse.c tool_formparse.h tool_getparam.c tool_getparam.h tool_getpass.c tool_getpass.h tool_help.c tool_help.h tool_helpers.c tool_helpers.h tool_hugehelp.h tool_ipfs.c tool_ipfs.h tool_libinfo.c tool_libinfo.h tool_listhelp.c tool_main.c tool_main.h tool_msgs.c tool_msgs.h tool_operate.c tool_operate.h tool_operhlp.c tool_operhlp.h tool_paramhlp.c tool_paramhlp.h tool_parsecfg.c tool_parsecfg.h tool_progress.c tool_progress.h tool_sdecls.h tool_setopt.c tool_setopt.h tool_setup.h tool_ssls.c tool_ssls.h tool_stderr.c tool_stderr.h tool_urlglob.c tool_urlglob.h tool_util.c tool_util.h tool_version.h tool_vms.c tool_vms.h tool_writeout.c tool_writeout.h tool_writeout_json.c tool_writeout_json.h tool_xattr.c tool_xattr.h var.c var.h
tests
certs .gitignore CMakeLists.txt Makefile.am Makefile.inc genserv.pl srp-verifier-conf srp-verifier-db test-ca.cnf test-ca.prm test-client-cert.prm test-client-eku-only.prm test-localhost-san-first.prm test-localhost-san-last.prm test-localhost.nn.prm test-localhost.prm test-localhost0h.prm
cmake CMakeLists.txt test.c test.cpp test.sh
data .gitignore DISABLED Makefile.am data-xml1 data1400.c data1401.c data1402.c data1403.c data1404.c data1405.c data1406.c data1407.c data1420.c data1461.txt data1463.txt data1465.c data1481.c data1705-1.md data1705-2.md data1705-3.md data1705-4.md data1705-stdout.1 data1706-1.md data1706-2.md data1706-3.md data1706-4.md data1706-stdout.txt data320.html test1 test10 test100 test1000 test1001 test1002 test1003 test1004 test1005 test1006 test1007 test1008 test1009 test101 test1010 test1011 test1012 test1013 test1014 test1015 test1016 test1017 test1018 test1019 test102 test1020 test1021 test1022 test1023 test1024 test1025 test1026 test1027 test1028 test1029 test103 test1030 test1031 test1032 test1033 test1034 test1035 test1036 test1037 test1038 test1039 test104 test1040 test1041 test1042 test1043 test1044 test1045 test1046 test1047 test1048 test1049 test105 test1050 test1051 test1052 test1053 test1054 test1055 test1056 test1057 test1058 test1059 test106 test1060 test1061 test1062 test1063 test1064 test1065 test1066 test1067 test1068 test1069 test107 test1070 test1071 test1072 test1073 test1074 test1075 test1076 test1077 test1078 test1079 test108 test1080 test1081 test1082 test1083 test1084 test1085 test1086 test1087 test1088 test1089 test109 test1090 test1091 test1092 test1093 test1094 test1095 test1096 test1097 test1098 test1099 test11 test110 test1100 test1101 test1102 test1103 test1104 test1105 test1106 test1107 test1108 test1109 test111 test1110 test1111 test1112 test1113 test1114 test1115 test1116 test1117 test1118 test1119 test112 test1120 test1121 test1122 test1123 test1124 test1125 test1126 test1127 test1128 test1129 test113 test1130 test1131 test1132 test1133 test1134 test1135 test1136 test1137 test1138 test1139 test114 test1140 test1141 test1142 test1143 test1144 test1145 test1146 test1147 test1148 test1149 test115 test1150 test1151 test1152 test1153 test1154 test1155 test1156 test1157 test1158 test1159 test116 test1160 test1161 test1162 test1163 test1164 test1165 test1166 test1167 test1168 test1169 test117 test1170 test1171 test1172 test1173 test1174 test1175 test1176 test1177 test1178 test1179 test118 test1180 test1181 test1182 test1183 test1184 test1185 test1186 test1187 test1188 test1189 test119 test1190 test1191 test1192 test1193 test1194 test1195 test1196 test1197 test1198 test1199 test12 test120 test1200 test1201 test1202 test1203 test1204 test1205 test1206 test1207 test1208 test1209 test121 test1210 test1211 test1212 test1213 test1214 test1215 test1216 test1217 test1218 test1219 test122 test1220 test1221 test1222 test1223 test1224 test1225 test1226 test1227 test1228 test1229 test123 test1230 test1231 test1232 test1233 test1234 test1235 test1236 test1237 test1238 test1239 test124 test1240 test1241 test1242 test1243 test1244 test1245 test1246 test1247 test1248 test1249 test125 test1250 test1251 test1252 test1253 test1254 test1255 test1256 test1257 test1258 test1259 test126 test1260 test1261 test1262 test1263 test1264 test1265 test1266 test1267 test1268 test1269 test127 test1270 test1271 test1272 test1273 test1274 test1275 test1276 test1277 test1278 test1279 test128 test1280 test1281 test1282 test1283 test1284 test1285 test1286 test1287 test1288 test1289 test129 test1290 test1291 test1292 test1293 test1294 test1295 test1296 test1297 test1298 test1299 test13 test130 test1300 test1301 test1302 test1303 test1304 test1305 test1306 test1307 test1308 test1309 test131 test1310 test1311 test1312 test1313 test1314 test1315 test1316 test1317 test1318 test1319 test132 test1320 test1321 test1322 test1323 test1324 test1325 test1326 test1327 test1328 test1329 test133 test1330 test1331 test1332 test1333 test1334 test1335 test1336 test1337 test1338 test1339 test134 test1340 test1341 test1342 test1343 test1344 test1345 test1346 test1347 test1348 test1349 test135 test1350 test1351 test1352 test1353 test1354 test1355 test1356 test1357 test1358 test1359 test136 test1360 test1361 test1362 test1363 test1364 test1365 test1366 test1367 test1368 test1369 test137 test1370 test1371 test1372 test1373 test1374 test1375 test1376 test1377 test1378 test1379 test138 test1380 test1381 test1382 test1383 test1384 test1385 test1386 test1387 test1388 test1389 test139 test1390 test1391 test1392 test1393 test1394 test1395 test1396 test1397 test1398 test1399 test14 test140 test1400 test1401 test1402 test1403 test1404 test1405 test1406 test1407 test1408 test1409 test141 test1410 test1411 test1412 test1413 test1414 test1415 test1416 test1417 test1418 test1419 test142 test1420 test1421 test1422 test1423 test1424 test1425 test1426 test1427 test1428 test1429 test143 test1430 test1431 test1432 test1433 test1434 test1435 test1436 test1437 test1438 test1439 test144 test1440 test1441 test1442 test1443 test1444 test1445 test1446 test1447 test1448 test1449 test145 test1450 test1451 test1452 test1453 test1454 test1455 test1456 test1457 test1458 test1459 test146 test1460 test1461 test1462 test1463 test1464 test1465 test1466 test1467 test1468 test1469 test147 test1470 test1471 test1472 test1473 test1474 test1475 test1476 test1477 test1478 test1479 test148 test1480 test1481 test1482 test1483 test1484 test1485 test1486 test1487 test1488 test1489 test149 test1490 test1491 test1492 test1493 test1494 test1495 test1496 test1497 test1498 test1499 test15 test150 test1500 test1501 test1502 test1503 test1504 test1505 test1506 test1507 test1508 test1509 test151 test1510 test1511 test1512 test1513 test1514 test1515 test1516 test1517 test1518 test1519 test152 test1520 test1521 test1522 test1523 test1524 test1525 test1526 test1527 test1528 test1529 test153 test1530 test1531 test1532 test1533 test1534 test1535 test1536 test1537 test1538 test1539 test154 test1540 test1541 test1542 test1543 test1544 test1545 test1546 test1547 test1548 test1549 test155 test1550 test1551 test1552 test1553 test1554 test1555 test1556 test1557 test1558 test1559 test156 test1560 test1561 test1562 test1563 test1564 test1565 test1566 test1567 test1568 test1569 test157 test1570 test1571 test1572 test1573 test1574 test1575 test1576 test1577 test1578 test1579 test158 test1580 test1581 test1582 test1583 test1584 test1585 test1586 test1587 test1588 test1589 test159 test1590 test1591 test1592 test1593 test1594 test1595 test1596 test1597 test1598 test1599 test16 test160 test1600 test1601 test1602 test1603 test1604 test1605 test1606 test1607 test1608 test1609 test161 test1610 test1611 test1612 test1613 test1614 test1615 test1616 test1617 test1618 test1619 test162 test1620 test1621 test1622 test1623 test1624 test1625 test1626 test1627 test1628 test1629 test163 test1630 test1631 test1632 test1633 test1634 test1635 test1636 test1637 test1638 test1639 test164 test1640 test1641 test1642 test1643 test1644 test1645 test165 test1650 test1651 test1652 test1653 test1654 test1655 test1656 test1657 test1658 test1659 test166 test1660 test1661 test1662 test1663 test1664 test1665 test1666 test1667 test1668 test1669 test167 test1670 test1671 test1672 test1673 test1674 test1675 test1676 test168 test1680 test1681 test1682 test1683 test1684 test1685 test169 test17 test170 test1700 test1701 test1702 test1703 test1704 test1705 test1706 test1707 test1708 test1709 test171 test1710 test1711 test1712 test1713 test1714 test1715 test172 test1720 test1721 test173 test174 test175 test176 test177 test178 test179 test18 test180 test1800 test1801 test1802 test181 test182 test183 test184 test1847 test1848 test1849 test185 test1850 test1851 test186 test187 test188 test189 test19 test190 test1900 test1901 test1902 test1903 test1904 test1905 test1906 test1907 test1908 test1909 test191 test1910 test1911 test1912 test1913 test1914 test1915 test1916 test1917 test1918 test1919 test192 test1920 test1921 test193 test1933 test1934 test1935 test1936 test1937 test1938 test1939 test194 test1940 test1941 test1942 test1943 test1944 test1945 test1946 test1947 test1948 test195 test1955 test1956 test1957 test1958 test1959 test196 test1960 test1964 test1965 test1966 test197 test1970 test1971 test1972 test1973 test1974 test1975 test1976 test1977 test1978 test1979 test198 test1980 test1981 test1982 test1983 test1984 test199 test2 test20 test200 test2000 test2001 test2002 test2003 test2004 test2005 test2006 test2007 test2008 test2009 test201 test2010 test2011 test2012 test2013 test2014 test202 test2023 test2024 test2025 test2026 test2027 test2028 test2029 test203 test2030 test2031 test2032 test2033 test2034 test2035 test2037 test2038 test2039 test204 test2040 test2041 test2042 test2043 test2044 test2045 test2046 test2047 test2048 test2049 test205 test2050 test2051 test2052 test2053 test2054 test2055 test2056 test2057 test2058 test2059 test206 test2060 test2061 test2062 test2063 test2064 test2065 test2066 test2067 test2068 test2069 test207 test2070 test2071 test2072 test2073 test2074 test2075 test2076 test2077 test2078 test2079 test208 test2080 test2081 test2082 test2083 test2084 test2085 test2086 test2087 test2088 test2089 test209 test2090 test2091 test2092 test21 test210 test2100 test2101 test2102 test2103 test2104 test211 test212 test213 test214 test215 test216 test217 test218 test219 test22 test220 test2200 test2201 test2202 test2203 test2204 test2205 test2206 test2207 test221 test222 test223 test224 test225 test226 test227 test228 test229 test23 test230 test2300 test2301 test2302 test2303 test2304 test2306 test2307 test2308 test2309 test231 test232 test233 test234 test235 test236 test237 test238 test239 test24 test240 test2400 test2401 test2402 test2403 test2404 test2405 test2406 test2407 test2408 test2409 test241 test2410 test2411 test242 test243 test244 test245 test246 test247 test248 test249 test25 test250 test2500 test2501 test2502 test2503 test2504 test2505 test2506 test251 test252 test253 test254 test255 test256 test257 test258 test259 test26 test260 test2600 test2601 test2602 test2603 test2604 test2605 test261 test262 test263 test264 test265 test266 test267 test268 test269 test27 test270 test2700 test2701 test2702 test2703 test2704 test2705 test2706 test2707 test2708 test2709 test271 test2710 test2711 test2712 test2713 test2714 test2715 test2716 test2717 test2718 test2719 test272 test2720 test2721 test2722 test2723 test273 test274 test275 test276 test277 test278 test279 test28 test280 test281 test282 test283 test284 test285 test286 test287 test288 test289 test29 test290 test291 test292 test293 test294 test295 test296 test297 test298 test299 test3 test30 test300 test3000 test3001 test3002 test3003 test3004 test3005 test3006 test3007 test3008 test3009 test301 test3010 test3011 test3012 test3013 test3014 test3015 test3016 test3017 test3018 test3019 test302 test3020 test3021 test3022 test3023 test3024 test3025 test3026 test3027 test3028 test3029 test303 test3030 test3031 test3032 test3033 test3034 test3035 test3036 test304 test305 test306 test307 test308 test309 test31 test310 test3100 test3101 test3102 test3103 test3104 test3105 test3106 test311 test312 test313 test314 test315 test316 test317 test318 test319 test32 test320 test3200 test3201 test3202 test3203 test3204 test3205 test3206 test3207 test3208 test3209 test321 test3210 test3211 test3212 test3213 test3214 test3215 test3216 test3217 test3218 test3219 test322 test3220 test323 test324 test325 test326 test327 test328 test329 test33 test330 test3300 test3301 test3302 test331 test332 test333 test334 test335 test336 test337 test338 test339 test34 test340 test341 test342 test343 test344 test345 test346 test347 test348 test349 test35 test350 test351 test352 test353 test354 test355 test356 test357 test358 test359 test36 test360 test361 test362 test363 test364 test365 test366 test367 test368 test369 test37 test370 test371 test372 test373 test374 test375 test376 test378 test379 test38 test380 test381 test383 test384 test385 test386 test387 test388 test389 test39 test390 test391 test392 test393 test394 test395 test396 test397 test398 test399 test4 test40 test400 test4000 test4001 test401 test402 test403 test404 test405 test406 test407 test408 test409 test41 test410 test411 test412 test413 test414 test415 test416 test417 test418 test419 test42 test420 test421 test422 test423 test424 test425 test426 test427 test428 test429 test43 test430 test431 test432 test433 test434 test435 test436 test437 test438 test439 test44 test440 test441 test442 test443 test444 test445 test446 test447 test448 test449 test45 test450 test451 test452 test453 test454 test455 test456 test457 test458 test459 test46 test460 test461 test462 test463 test467 test468 test469 test47 test470 test471 test472 test473 test474 test475 test476 test477 test478 test479 test48 test480 test481 test482 test483 test484 test485 test486 test487 test488 test489 test49 test490 test491 test492 test493 test494 test495 test496 test497 test498 test499 test5 test50 test500 test501 test502 test503 test504 test505 test506 test507 test508 test509 test51 test510 test511 test512 test513 test514 test515 test516 test517 test518 test519 test52 test520 test521 test522 test523 test524 test525 test526 test527 test528 test529 test53 test530 test531 test532 test533 test534 test535 test536 test537 test538 test539 test54 test540 test541 test542 test543 test544 test545 test546 test547 test548 test549 test55 test550 test551 test552 test553 test554 test555 test556 test557 test558 test559 test56 test560 test561 test562 test563 test564 test565 test566 test567 test568 test569 test57 test570 test571 test572 test573 test574 test575 test576 test577 test578 test579 test58 test580 test581 test582 test583 test584 test585 test586 test587 test588 test589 test59 test590 test591 test592 test593 test594 test595 test596 test597 test598 test599 test6 test60 test600 test601 test602 test603 test604 test605 test606 test607 test608 test609 test61 test610 test611 test612 test613 test614 test615 test616 test617 test618 test619 test62 test620 test621 test622 test623 test624 test625 test626 test627 test628 test629 test63 test630 test631 test632 test633 test634 test635 test636 test637 test638 test639 test64 test640 test641 test642 test643 test644 test645 test646 test647 test648 test649 test65 test650 test651 test652 test653 test654 test655 test656 test658 test659 test66 test660 test661 test662 test663 test664 test665 test666 test667 test668 test669 test67 test670 test671 test672 test673 test674 test675 test676 test677 test678 test679 test68 test680 test681 test682 test683 test684 test685 test686 test687 test688 test689 test69 test690 test691 test692 test693 test694 test695 test696 test697 test698 test699 test7 test70 test700 test701 test702 test703 test704 test705 test706 test707 test708 test709 test71 test710 test711 test712 test713 test714 test715 test716 test717 test718 test719 test72 test720 test721 test722 test723 test724 test725 test726 test727 test728 test729 test73 test730 test731 test732 test733 test734 test735 test736 test737 test738 test739 test74 test740 test741 test742 test743 test744 test745 test746 test747 test748 test749 test75 test750 test751 test752 test753 test754 test755 test756 test757 test758 test759 test76 test760 test761 test762 test763 test764 test765 test766 test767 test768 test769 test77 test770 test771 test772 test773 test774 test775 test776 test777 test778 test779 test78 test780 test781 test782 test783 test784 test785 test786 test787 test788 test789 test79 test790 test791 test792 test793 test794 test795 test796 test797 test798 test799 test8 test80 test800 test801 test802 test803 test804 test805 test806 test807 test808 test809 test81 test810 test811 test812 test813 test814 test815 test816 test817 test818 test819 test82 test820 test821 test822 test823 test824 test825 test826 test827 test828 test829 test83 test830 test831 test832 test833 test834 test835 test836 test837 test838 test839 test84 test840 test841 test842 test843 test844 test845 test846 test847 test848 test849 test85 test850 test851 test852 test853 test854 test855 test856 test857 test858 test859 test86 test860 test861 test862 test863 test864 test865 test866 test867 test868 test869 test87 test870 test871 test872 test873 test874 test875 test876 test877 test878 test879 test88 test880 test881 test882 test883 test884 test885 test886 test887 test888 test889 test89 test890 test891 test892 test893 test894 test895 test896 test897 test898 test899 test9 test90 test900 test901 test902 test903 test904 test905 test906 test907 test908 test909 test91 test910 test911 test912 test913 test914 test915 test916 test917 test918 test919 test92 test920 test921 test922 test923 test924 test925 test926 test927 test928 test929 test93 test930 test931 test932 test933 test934 test935 test936 test937 test938 test939 test94 test940 test941 test942 test943 test944 test945 test946 test947 test948 test949 test95 test950 test951 test952 test953 test954 test955 test956 test957 test958 test959 test96 test960 test961 test962 test963 test964 test965 test966 test967 test968 test969 test97 test970 test971 test972 test973 test974 test975 test976 test977 test978 test979 test98 test980 test981 test982 test983 test984 test985 test986 test987 test988 test989 test99 test990 test991 test992 test993 test994 test995 test996 test997 test998 test999
http
testenv
mod_curltest .gitignore mod_curltest.c
__init__.py caddy.py certs.py client.py curl.py dante.py dnsd.py env.py httpd.py nghttpx.py ports.py sshd.py vsftpd.py ws_echo_server.py
.gitignore CMakeLists.txt Makefile.am config.ini.in conftest.py requirements.txt scorecard.py test_01_basic.py test_02_download.py test_03_goaway.py test_04_stuttered.py test_05_errors.py test_06_eyeballs.py test_07_upload.py test_08_caddy.py test_09_push.py test_10_proxy.py test_11_unix.py test_12_reuse.py test_13_proxy_auth.py test_14_auth.py test_15_tracing.py test_16_info.py test_17_ssl_use.py test_18_methods.py test_19_shutdown.py test_20_websockets.py test_21_resolve.py test_22_httpsrr.py test_30_vsftpd.py test_31_vsftpds.py test_32_ftps_vsftpd.py test_40_socks.py test_50_scp.py test_51_sftp.py
libtest .gitignore CMakeLists.txt Makefile.am Makefile.inc cli_ftp_upload.c cli_h2_pausing.c cli_h2_serverpush.c cli_h2_upgrade_extreme.c cli_hx_download.c cli_hx_upload.c cli_tls_session_reuse.c cli_upload_pausing.c cli_ws_data.c cli_ws_pingpong.c first.c first.h lib1156.c lib1301.c lib1308.c lib1485.c lib1500.c lib1501.c lib1502.c lib1506.c lib1507.c lib1508.c lib1509.c lib1510.c lib1511.c lib1512.c lib1513.c lib1514.c lib1515.c lib1517.c lib1518.c lib1520.c lib1522.c lib1523.c lib1525.c lib1526.c lib1527.c lib1528.c lib1529.c lib1530.c lib1531.c lib1532.c lib1533.c lib1534.c lib1535.c lib1536.c lib1537.c lib1538.c lib1540.c lib1541.c lib1542.c lib1545.c lib1549.c lib1550.c lib1551.c lib1552.c lib1553.c lib1554.c lib1555.c lib1556.c lib1557.c lib1558.c lib1559.c lib1560.c lib1564.c lib1565.c lib1567.c lib1568.c lib1569.c lib1571.c lib1576.c lib1582.c lib1587.c lib1588.c lib1589.c lib1591.c lib1592.c lib1593.c lib1594.c lib1597.c lib1598.c lib1599.c lib1662.c lib1900.c lib1901.c lib1902.c lib1903.c lib1905.c lib1906.c lib1907.c lib1908.c lib1910.c lib1911.c lib1912.c lib1913.c lib1915.c lib1916.c lib1918.c lib1919.c lib1920.c lib1921.c lib1933.c lib1934.c lib1935.c lib1936.c lib1937.c lib1938.c lib1939.c lib1940.c lib1945.c lib1947.c lib1948.c lib1955.c lib1956.c lib1957.c lib1958.c lib1959.c lib1960.c lib1964.c lib1965.c lib1970.c lib1971.c lib1972.c lib1973.c lib1974.c lib1975.c lib1977.c lib1978.c lib2023.c lib2032.c lib2082.c lib2301.c lib2302.c lib2304.c lib2306.c lib2308.c lib2309.c lib2402.c lib2404.c lib2405.c lib2502.c lib2504.c lib2505.c lib2506.c lib2700.c lib3010.c lib3025.c lib3026.c lib3027.c lib3033.c lib3034.c lib3100.c lib3101.c lib3102.c lib3103.c lib3104.c lib3105.c lib3207.c lib3208.c lib500.c lib501.c lib502.c lib503.c lib504.c lib505.c lib506.c lib507.c lib508.c lib509.c lib510.c lib511.c lib512.c lib513.c lib514.c lib515.c lib516.c lib517.c lib518.c lib519.c lib520.c lib521.c lib523.c lib524.c lib525.c lib526.c lib530.c lib533.c lib536.c lib537.c lib539.c lib540.c lib541.c lib542.c lib543.c lib544.c lib547.c lib549.c lib552.c lib553.c lib554.c lib555.c lib556.c lib557.c lib558.c lib559.c lib560.c lib562.c lib564.c lib566.c lib567.c lib568.c lib569.c lib570.c lib571.c lib572.c lib573.c lib574.c lib575.c lib576.c lib578.c lib579.c lib582.c lib583.c lib586.c lib589.c lib590.c lib591.c lib597.c lib598.c lib599.c lib643.c lib650.c lib651.c lib652.c lib653.c lib654.c lib655.c lib658.c lib659.c lib661.c lib666.c lib667.c lib668.c lib670.c lib674.c lib676.c lib677.c lib678.c lib694.c lib695.c lib751.c lib753.c lib757.c lib758.c lib766.c memptr.c mk-lib1521.pl test1013.pl test1022.pl test307.pl test610.pl test613.pl testtrace.c testtrace.h testutil.c testutil.h unitcheck.h
server .checksrc .gitignore CMakeLists.txt Makefile.am Makefile.inc dnsd.c first.c first.h getpart.c mqttd.c resolve.c rtspd.c sockfilt.c socksd.c sws.c tftpd.c util.c
tunit .gitignore CMakeLists.txt Makefile.am Makefile.inc README.md tool1394.c tool1604.c tool1621.c tool1622.c tool1623.c tool1720.c
unit .gitignore CMakeLists.txt Makefile.am Makefile.inc README.md unit1300.c unit1302.c unit1303.c unit1304.c unit1305.c unit1307.c unit1309.c unit1323.c unit1330.c unit1395.c unit1396.c unit1397.c unit1398.c unit1399.c unit1600.c unit1601.c unit1602.c unit1603.c unit1605.c unit1606.c unit1607.c unit1608.c unit1609.c unit1610.c unit1611.c unit1612.c unit1614.c unit1615.c unit1616.c unit1620.c unit1625.c unit1626.c unit1627.c unit1636.c unit1650.c unit1651.c unit1652.c unit1653.c unit1654.c unit1655.c unit1656.c unit1657.c unit1658.c unit1660.c unit1661.c unit1663.c unit1664.c unit1666.c unit1667.c unit1668.c unit1669.c unit1674.c unit1675.c unit1676.c unit1979.c unit1980.c unit2600.c unit2601.c unit2602.c unit2603.c unit2604.c unit2605.c unit3200.c unit3205.c unit3211.c unit3212.c unit3213.c unit3214.c unit3216.c unit3219.c unit3300.c unit3301.c unit3302.c
.gitignore CMakeLists.txt Makefile.am allversions.pm appveyor.pm azure.pm config.in configurehelp.pm.in devtest.pl dictserver.py directories.pm ech_combos.py ech_tests.sh ftpserver.pl getpart.pm globalconfig.pm http-server.pl http2-server.pl http3-server.pl memanalyze.pl memanalyzer.pm negtelnetserver.py nghttpx.conf pathhelp.pm processhelp.pm requirements.txt rtspserver.pl runner.pm runtests.pl secureserver.pl serverhelp.pm servers.pm smbserver.py sshhelp.pm sshserver.pl test1119.pl test1135.pl test1139.pl test1140.pl test1165.pl test1167.pl test1173.pl test1175.pl test1177.pl test1222.pl test1275.pl test1276.pl test1477.pl test1486.pl test1488.pl test1544.pl test1707.pl test745.pl test971.pl testcurl.pl testutil.pm tftpserver.pl util.py valgrind.pm valgrind.supp
.clang-tidy.yml .dir-locals.el .editorconfig .git-blame-ignore-revs .gitattributes .gitignore .mailmap CHANGES.md CMakeLists.txt COPYING Dockerfile GIT-INFO.md Makefile.am README README.md RELEASE-NOTES REUSE.toml SECURITY.md acinclude.m4 appveyor.sh appveyor.yml configure.ac curl-config.in libcurl.pc.in renovate.json
examples .env config.ini crypto_test.lua env_test.lua fs_example.lua http_server.lua https_test.lua ini_example.lua json.lua log.lua path_fs_example.lua process_example.lua request_download.lua request_test.lua run_all.lua sqlite_example.lua sqlite_http_template.lua stash_test.lua template_test.lua timer.lua websocket.lua
iniparser
.github
ISSUE_TEMPLATE config.yml
workflows disable-pull-requests.yml trigger-gitlab-ci.yml
cmake JoinPaths.cmake config.cmake.in pc.in
example iniexample.c iniwrite.c parse.c twisted-errors.ini twisted-genhuge.py twisted-ofkey.ini twisted-ofval.ini twisted.ini
src dictionary.c dictionary.h iniparser.c iniparser.h
test
ressources
bad_ini ends_well.ini twisted-errors.ini twisted-ofkey.ini twisted-ofval.ini
good_ini empty.ini spaced.ini spaced2.ini twisted.ini
gruezi.ini old.ini quotes.ini utf8.ini
CMakeLists.txt test_dictionary.c test_iniparser.c unity-config.yml unity_config.h
.cmake-format.py .gitignore .gitlab-ci.yml .gitmessage .travis.yml AUTHORS CMakeLists.txt FAQ-en.md FAQ-zhcn.md INSTALL LICENSE README.md compile_commands.json
jinjac
example CMakeLists.txt example.c
jinjac_test_app CMakeLists.txt jinjac_test_app.c
libjinjac
include jinjac.h
src CMakeLists.txt ast.c ast.h block_statement.c block_statement.h buffer.c buffer.h buildin.c buildin.h common.h convert.c convert.h flex_decl.h jfunction.c jfunction.h jinja_expression.l jinja_expression.y jinjac_parse.c jinjac_parse.h jinjac_stream.c jinjac_stream.h jlist.c jlist.h jobject.c jobject.h parameter.c parameter.h str_obj.c str_obj.h trace.c trace.h
CMakeLists.txt
test .gitignore CMakeLists.txt autotest.rb test_01.expected test_01.jinja test_01b.expected test_01b.jinja test_01c.expected test_01c.jinja test_01d.expected test_01d.jinja test_02.expected test_02.jinja test_03.expected test_03.jinja test_04.expected test_04.jinja test_05.expected test_05.jinja test_06.expected test_06.jinja test_07.expected test_07.jinja test_08.expected test_08.jinja test_08b.expected test_08b.jinja test_09.expected test_09.jinja test_10.expected test_10.jinja test_11.expected test_11.jinja test_12.expected test_12.jinja test_13.expected test_13.jinja test_14.expected test_14.jinja test_15.expected test_15.jinja test_16.expected test_16.jinja test_17.expected test_17.jinja test_18.expected test_18.jinja test_18b.expected test_18b.jinja test_18c.expected test_18c.jinja test_19.expected test_19.jinja test_19b.expected test_19b.jinja test_19c.expected test_19c.jinja test_19d.expected test_19d.jinja test_19e.expected test_19e.jinja test_19f.expected test_19f.jinja test_20.expected test_20.jinja test_21.expected test_21.jinja test_22.expected test_22.jinja test_22a.expected test_22a.jinja test_22b.expected test_22b.jinja test_23.expected test_23.jinja test_24.expected test_24.jinja
.gitignore CMakeLists.txt LICENSE.txt README.md build_coverage.sh build_debug.sh build_release.sh cppcheck_analysis.sh
libev Changes LICENSE Makefile Makefile.am Makefile.in README Symbols.ev Symbols.event aclocal.m4 autogen.sh compile config.guess config.h config.h.in config.status config.sub configure configure.ac depcomp ev++.h ev.3 ev.c ev.h ev.pod ev_epoll.c ev_kqueue.c ev_poll.c ev_port.c ev_select.c ev_vars.h ev_win32.c ev_wrap.h event.c event.h install-sh libev.m4 libtool ltmain.sh missing mkinstalldirs stamp-h1
luajit
doc
img contact.png
bluequad-print.css bluequad.css contact.html ext_buffer.html ext_c_api.html ext_ffi.html ext_ffi_api.html ext_ffi_semantics.html ext_ffi_tutorial.html ext_jit.html ext_profiler.html extensions.html install.html luajit.html running.html
dynasm dasm_arm.h dasm_arm.lua dasm_arm64.h dasm_arm64.lua dasm_mips.h dasm_mips.lua dasm_mips64.lua dasm_ppc.h dasm_ppc.lua dasm_proto.h dasm_x64.lua dasm_x86.h dasm_x86.lua dynasm.lua
etc luajit.1 luajit.pc
src
host .gitignore README buildvm.c buildvm.h buildvm_asm.c buildvm_fold.c buildvm_lib.c buildvm_libbc.h buildvm_peobj.c genlibbc.lua genminilua.lua genversion.lua minilua.c
jit .gitignore bc.lua bcsave.lua dis_arm.lua dis_arm64.lua dis_arm64be.lua dis_mips.lua dis_mips64.lua dis_mips64el.lua dis_mips64r6.lua dis_mips64r6el.lua dis_mipsel.lua dis_ppc.lua dis_x64.lua dis_x86.lua dump.lua p.lua v.lua zone.lua
.gitignore Makefile Makefile.dep lauxlib.h lib_aux.c lib_base.c lib_bit.c lib_buffer.c lib_debug.c lib_ffi.c lib_init.c lib_io.c lib_jit.c lib_math.c lib_os.c lib_package.c lib_string.c lib_table.c lj_alloc.c lj_alloc.h lj_api.c lj_arch.h lj_asm.c lj_asm.h lj_asm_arm.h lj_asm_arm64.h lj_asm_mips.h lj_asm_ppc.h lj_asm_x86.h lj_assert.c lj_bc.c lj_bc.h lj_bcdump.h lj_bcread.c lj_bcwrite.c lj_buf.c lj_buf.h lj_carith.c lj_carith.h lj_ccall.c lj_ccall.h lj_ccallback.c lj_ccallback.h lj_cconv.c lj_cconv.h lj_cdata.c lj_cdata.h lj_char.c lj_char.h lj_clib.c lj_clib.h lj_cparse.c lj_cparse.h lj_crecord.c lj_crecord.h lj_ctype.c lj_ctype.h lj_debug.c lj_debug.h lj_def.h lj_dispatch.c lj_dispatch.h lj_emit_arm.h lj_emit_arm64.h lj_emit_mips.h lj_emit_ppc.h lj_emit_x86.h lj_err.c lj_err.h lj_errmsg.h lj_ff.h lj_ffrecord.c lj_ffrecord.h lj_frame.h lj_func.c lj_func.h lj_gc.c lj_gc.h lj_gdbjit.c lj_gdbjit.h lj_ir.c lj_ir.h lj_ircall.h lj_iropt.h lj_jit.h lj_lex.c lj_lex.h lj_lib.c lj_lib.h lj_load.c lj_mcode.c lj_mcode.h lj_meta.c lj_meta.h lj_obj.c lj_obj.h lj_opt_dce.c lj_opt_fold.c lj_opt_loop.c lj_opt_mem.c lj_opt_narrow.c lj_opt_sink.c lj_opt_split.c lj_parse.c lj_parse.h lj_prng.c lj_prng.h lj_profile.c lj_profile.h lj_record.c lj_record.h lj_serialize.c lj_serialize.h lj_snap.c lj_snap.h lj_state.c lj_state.h lj_str.c lj_str.h lj_strfmt.c lj_strfmt.h lj_strfmt_num.c lj_strscan.c lj_strscan.h lj_tab.c lj_tab.h lj_target.h lj_target_arm.h lj_target_arm64.h lj_target_mips.h lj_target_ppc.h lj_target_x86.h lj_trace.c lj_trace.h lj_traceerr.h lj_udata.c lj_udata.h lj_vm.h lj_vmevent.c lj_vmevent.h lj_vmmath.c ljamalg.c lua.h lua.hpp luaconf.h luajit.c luajit_rolling.h lualib.h msvcbuild.bat nxbuild.bat ps4build.bat ps5build.bat psvitabuild.bat vm_arm.dasc vm_arm64.dasc vm_mips.dasc vm_mips64.dasc vm_ppc.dasc vm_x64.dasc vm_x86.dasc xb1build.bat xedkbuild.bat
.gitattributes .gitignore .relver COPYRIGHT Makefile README
sqlite shell.c sqlite3.c sqlite3.h sqlite3ext.h
wolfssl
.github
ISSUE_TEMPLATE bug_report.yaml other.yaml
actions
install-apt-deps action.yml
scripts
zephyr-4.x external_libc.conf zephyr-test.sh
openssl-ech.sh tls-anvil-test.sh
workflows
disabled haproxy.yml hitch.yml hostap.yml
hostap-files
configs
07c9f183ea744ac04585fb6dd10220c75a5e2e74 hostapd.config tests wpa_supplicant.config
b607d2723e927a3446d89aed813f1aa6068186bb hostapd.config tests wpa_supplicant.config
hostap_2_10 extra.patch hostapd.config tests wpa_supplicant.config
Makefile README dbus-wpa_supplicant.conf
ada.yml arduino.yml async-examples.yml async.yml atecc608-sim.yml bind.yml cmake-autoconf.yml cmake.yml codespell.yml coverity-scan-fixes.yml cryptocb-only.yml curl.yml cyrus-sasl.yml disable-pk-algs.yml docker-Espressif.yml docker-OpenWrt.yml emnet-nonblock.yml fil-c.yml freertos-mem-track.yml gencertbuf.yml grpc.yml haproxy.yml hostap-vm.yml intelasm-c-fallback.yml ipmitool.yml jwt-cpp.yml krb5.yml libspdm.yml libssh2.yml libvncserver.yml linuxkm.yml macos-apple-native-cert-validation.yml mbedtls.sh mbedtls.yml membrowse-comment.yml membrowse-onboard.yml membrowse-report.yml memcached.sh memcached.yml mono.yml mosquitto.yml msmtp.yml msys2.yml multi-arch.yml multi-compiler.yml net-snmp.yml nginx.yml no-malloc.yml no-tls.yml nss.sh nss.yml ntp.yml ocsp.yml openldap.yml openssh.yml openssl-ech.yml opensslcoexist.yml openvpn.yml os-check.yml packaging.yml pam-ipmi.yml pq-all.yml pr-commit-check.yml psk.yml puf.yml python.yml rng-tools.yml rust-wrapper.yml se050-sim.yml smallStackSize.yml socat.yml softhsm.yml sssd.yml stm32-sim.yml stsafe-a120-sim.yml stunnel.yml symbol-prefixes.yml threadx.yml tls-anvil.yml trackmemory.yml watcomc.yml win-csharp-test.yml wolfCrypt-Wconversion.yml wolfboot-integration.yml wolfsm.yml xcode.yml zephyr-4.x.yml zephyr.yml
PULL_REQUEST_TEMPLATE.md SECURITY.md membrowse-targets.json
Docker
OpenWrt Dockerfile README.md runTests.sh
packaging
debian Dockerfile
fedora Dockerfile
wolfCLU Dockerfile README.md
yocto Dockerfile buildAndPush.sh
Dockerfile Dockerfile.cross-compiler README.md buildAndPush.sh include.am run.sh
IDE
ARDUINO
sketches
wolfssl_client README.md
wolfssl_server README.md
wolfssl_version README.md
README.md
Arduino_README_prepend.md README.md include.am keywords.txt library.properties.template wolfssl-arduino.cpp wolfssl-arduino.sh wolfssl.h
AURIX Cpu0_Main.c README.md include.am user_settings.h wolf_main.c
Android Android.bp README.md include.am user_settings.h
CRYPTOCELL README.md include.am main.c user_settings.h
CSBENCH include.am user_settings.h
ECLIPSE
DEOS
deos_wolfssl .options
README.md deos_malloc.c include.am tls_wolfssl.c tls_wolfssl.h user_settings.h
MICRIUM README.md client_wolfssl.c client_wolfssl.h include.am server_wolfssl.c server_wolfssl.h user_settings.h wolfsslRunTests.c
RTTHREAD README.md include.am user_settings.h wolfssl_test.c
SIFIVE README.md include.am
Espressif
ESP-IDF
examples
template
VisualGDB wolfssl_template_IDF_v5.1_ESP32.vgdbproj
components
wolfssl
include user_settings.h
CMakeLists.txt Kconfig README.md component.mk
main
include main.h
CMakeLists.txt Kconfig.projbuild component.mk main.c
CMakeLists.txt Makefile README.md partitions_singleapp_large.csv sdkconfig.defaults sdkconfig.defaults.esp8266
wolfssl_benchmark
VisualGDB wolfssl_benchmark_IDF_v4.4_ESP32.sln wolfssl_benchmark_IDF_v4.4_ESP32.vgdbproj wolfssl_benchmark_IDF_v5_ESP32.sln wolfssl_benchmark_IDF_v5_ESP32.vgdbproj wolfssl_benchmark_IDF_v5_ESP32C3.sln wolfssl_benchmark_IDF_v5_ESP32C3.vgdbproj wolfssl_benchmark_IDF_v5_ESP32S3.sln wolfssl_benchmark_IDF_v5_ESP32S3.vgdbproj
components
wolfssl
include user_settings.h
CMakeLists.txt Kconfig README.md component.mk
main
include main.h
CMakeLists.txt Kconfig.projbuild component.mk main.c
CMakeLists.txt Makefile README.md partitions_singleapp_large.csv sdkconfig.defaults sdkconfig.defaults.esp8266
wolfssl_client
VisualGDB README.md wolfssl_client_IDF_v5_ESP32.sln wolfssl_client_IDF_v5_ESP32.vgdbproj
components
wolfssl
include user_settings.h
CMakeLists.txt Kconfig README.md component.mk
main
include client-tls.h main.h time_helper.h wifi_connect.h
CMakeLists.txt Kconfig.projbuild client-tls.c component.mk main.c time_helper.c wifi_connect.c
CMakeLists.txt Makefile README.md README_server_sm.md partitions_singleapp_large.csv sdkconfig.defaults sdkconfig.defaults.esp32c2 sdkconfig.defaults.esp8266 wolfssl_client_ESP8266.vgdbproj
wolfssl_server
VisualGDB README.md wolfssl_server_IDF_v5_ESP32.sln wolfssl_server_IDF_v5_ESP32.vgdbproj
components
wolfssl
include user_settings.h
CMakeLists.txt Kconfig README.md component.mk
main
include main.h server-tls.h time_helper.h wifi_connect.h
CMakeLists.txt Kconfig.projbuild component.mk main.c server-tls.c time_helper.c wifi_connect.c
CMakeLists.txt Makefile README.md README_server_sm.md partitions_singleapp_large.csv sdkconfig.defaults sdkconfig.defaults.esp32c2 sdkconfig.defaults.esp8266 wolfssl_server_ESP8266.vgdbproj
wolfssl_test
VisualGDB wolfssl_test-IDF_v5_ESP32.sln wolfssl_test-IDF_v5_ESP32.vgdbproj wolfssl_test-IDF_v5_ESP32C3.sln wolfssl_test-IDF_v5_ESP32C3.vgdbproj wolfssl_test-IDF_v5_ESP32C6.sln wolfssl_test-IDF_v5_ESP32C6.vgdbproj wolfssl_test_IDF_v5_ESP32S3.sln wolfssl_test_IDF_v5_ESP32S3.vgdbproj
components
wolfssl
include user_settings.h
CMakeLists.txt Kconfig README.md component.mk
main
include main.h
CMakeLists.txt Kconfig.projbuild component.mk main.c
CMakeLists.txt Makefile README.md partitions_singleapp_large.csv sdkconfig.defaults sdkconfig.defaults.esp32 sdkconfig.defaults.esp32c3 sdkconfig.defaults.esp32c6 sdkconfig.defaults.esp32h2 sdkconfig.defaults.esp32s2 sdkconfig.defaults.esp32s3 sdkconfig.defaults.esp8266 testAll.sh testMonitor.sh wolfssl_test_ESP8266.sln wolfssl_test_ESP8266.vgdbproj
wolfssl_test_idf
VisualGDB VisualGDB_wolfssl_test_idf.sln VisualGDB_wolfssl_test_idf.vgdbproj
main CMakeLists.txt Kconfig.projbuild component.mk main.c main_wip.c.ex time_helper.c time_helper.h
CMakeLists.txt Kconfig.projbuild README.md component.mk sdkconfig.defaults
README.md
libs CMakeLists.txt README.md component.mk tigard.cfg
test CMakeLists.txt README.md component.mk test_wolfssl.c
README.md README_32se.md UPDATE.md compileAllExamples.sh dummy_config_h dummy_test_paths.h setup.sh setup_win.bat user_settings.h
README.md include.am
GCC-ARM
Header user_settings.h
Source armtarget.c benchmark_main.c test_main.c tls_client.c tls_server.c wolf_main.c
Makefile Makefile.bench Makefile.client Makefile.common Makefile.server Makefile.static Makefile.test README.md include.am linker.ld linker_fips.ld
Gaisler-BCC README.md include.am
HEXAGON
DSP Makefile wolfssl_dsp.idl
Makefile README.md build.sh ecc-verify-benchmark.c ecc-verify.c include.am user_settings.h
HEXIWEAR
wolfSSL_HW .cwGeneratedFileSetLog user_settings.h
IAR-EWARM
Projects
benchmark benchmark-main.c current_time.c wolfCrypt-benchmark.ewd wolfCrypt-benchmark.ewp
common minimum-startup.c wolfssl.icf
lib wolfSSL-Lib.ewd wolfSSL-Lib.ewp
test test-main.c wolfCrypt-test.ewd wolfCrypt-test.ewp
user_settings.h wolfssl.eww
embOS
SAMV71_XULT
embOS_SAMV71_XULT_Linker_Script samv71q21_wolfssl.icf
embOS_SAMV71_XULT_user_settings user_settings.h user_settings_simple_example.h user_settings_verbose_example.h
embOS_wolfcrypt_benchmark_SAMV71_XULT
Application runBenchmarks.c
README_wolfcrypt_benchmark wolfcrypt_benchmark.ewd wolfcrypt_benchmark.ewp
embOS_wolfcrypt_lib_SAMV71_XULT README_wolfcrypt_lib wolfcrypt_lib.ewd wolfcrypt_lib.ewp
embOS_wolfcrypt_test_SAMV71_XULT
Application runWolfcryptTests.c
README_wolfcrypt_test wolfcrypt_test.ewd wolfcrypt_test.ewp
README_SAMV71
custom_port
custom_port_Linker_Script samv71q21_wolfssl.icf
custom_port_user_settings user_settings.h
wolfcrypt_benchmark_custom_port
Application runBenchmarks.c
wolfcrypt_test_custom_port
Application runWolfcryptTests.c
README_custom_port
extract_trial_here README_extract_trial_here
README
.gitignore README
IAR-MSP430 Makefile README.md include.am main.c user_settings.h
INTIME-RTOS Makefile README.md include.am libwolfssl.c libwolfssl.vcxproj user_settings.h wolfExamples.c wolfExamples.h wolfExamples.sln wolfExamples.vcxproj wolfssl-lib.sln wolfssl-lib.vcxproj
Infineon README.md include.am user_settings.h
KDS
config user_settings.h
include.am
LINUX-SGX README.md build.sh clean.sh include.am sgx_t_static.mk
LPCXPRESSO
lib_wolfssl lpc_18xx_port.c user_settings.h
wolf_example
src lpc_18xx_startup.c wolfssl_example.c
readme.txt
README.md
M68K
benchmark Makefile main.cpp
testwolfcrypt Makefile main.cpp
Makefile README.md include.am user_settings.h
MCUEXPRESSO
RT1170 fsl_caam_c.patch fsl_caam_h.patch user_settings.h
benchmark
source run_benchmark.c
wolfssl liblinks.xml
README.md include.am user_settings.h wolfcrypt_test.c
MDK-ARM
LPC43xx time-LCP43xx.c
MDK-ARM
wolfSSL Retarget.c cert_data.c cert_data.h config-BARE-METAL.h config-FS.h config-RTX-TCP-FS.h config-WOLFLIB.h main.c shell.c time-CortexM3-4.c time-dummy.c wolfssl_MDK_ARM.c wolfssl_MDK_ARM.h
STM32F2xx_StdPeriph_Lib time-STM32F2xx.c
MDK5-ARM
Conf user_settings.h
Inc wolfssl_MDK_ARM.h
Projects
CryptBenchmark Abstract.txt CryptBenchmark.sct CryptBenchmark.uvoptx CryptBenchmark.uvprojx main.c
CryptTest Abstract.txt CryptTest.sct CryptTest.uvoptx CryptTest.uvprojx main.c
EchoClient Abstract.txt EchoClient.uvoptx EchoClient.uvprojx main.c wolfssl-link.sct
EchoServer Abstract.txt EchoServer.uvoptx EchoServer.uvprojx main.c wolfssl-link.sct
SimpleClient Abstract.txt SimpleClient.uvoptx SimpleClient.uvprojx main.c wolfssl-link.sct
SimpleServer Abstract.txt SimpleServer.uvoptx SimpleServer.uvprojx main.c wolfssl-link.sct
wolfSSL-Full Abstract.txt main.c shell.c time-CortexM3-4.c wolfsslFull.uvoptx wolfsslFull.uvprojx
wolfSSL-Lib Abstract.txt wolfSSL-Lib.uvoptx wolfSSL-Lib.uvprojx
Src ssl-dummy.c
README.md include.am
MPLABX16
wolfcrypt_test.X
nbproject
private configurations.xml private.xml
configurations.xml include.am project.xml
Makefile
wolfssl.X
nbproject configurations.xml include.am project.xml
Makefile
README.md include.am main.c user_settings.h
MQX Makefile README-jp.md README.md client-tls.c include.am server-tls.c user_config.h user_settings.h
MSVS-2019-AZSPHERE
client client.c client.h
server server.c server.h
shared util.h
wolfssl_new_azsphere
HardwareDefinitions
avnet_mt3620_sk
inc
hw template_appliance.h
template_appliance.json
mt3620_rdb
inc
hw template_appliance.h
template_appliance.json
seeed_mt3620_mdb
inc
hw template_appliance.h
template_appliance.json
.gitignore CMakeLists.txt CMakeSettings.json app_manifest.json applibs_versions.h launch.vs.json main.c
README.md include.am user_settings.h
MYSQL CMakeLists_wolfCrypt.txt CMakeLists_wolfSSL.txt do.sh
NDS README.md
NETOS Makefile.wolfcrypt.inc README.md include.am user_settings.h user_settings.h-cert2425 user_settings.h-cert3389 wolfssl_netos_custom.c
OPENSTM32 README.md
PlatformIO
examples
wolfssl_benchmark
include README main.h
lib README
src CMakeLists.txt main.c
test README
CMakeLists.txt README.md platformio.ini sdkconfig.defaults wolfssl_benchmark.code-workspace
wolfssl_test
include README main.h
lib README
src CMakeLists.txt main.c
test README
CMakeLists.txt README.md platformio.ini sdkconfig.defaults wolfssl_test.code-workspace
README.md wolfssl_platformio.code-workspace
README.md include.am
QNX
CAAM-DRIVER Makefile
example-client Makefile client-tls.c
example-cmac Makefile cmac-test.c
example-server Makefile server-tls.c
testwolfcrypt Makefile
wolfssl Makefile user_settings.h
README.md include.am
RISCV
SIFIVE-HIFIVE1 Makefile README.md include.am main.c user_settings.h
SIFIVE-UNLEASHED README.md include.am
include.am
ROWLEY-CROSSWORKS-ARM Kinetis_FlashPlacement.xml README.md arm_startup.c benchmark_main.c hw.h include.am kinetis_hw.c retarget.c test_main.c user_settings.h wolfssl.hzp wolfssl_ltc.hzp
Renesas
cs+
Projects
common strings.h unistd.h user_settings.h wolfssl_dummy.c
t4_demo README_en.txt README_jp.txt t4_demo.mtpj wolf_client.c wolf_demo.h wolf_main.c wolf_server.c
test test.mtpj test_main.c
wolfssl_lib wolfssl_lib.mtpj
README include.am
e2studio
DK-S7G2
benchmark-template
src app_entry.c
example_server-template
src app_entry.c
wolfcrypttest-template
src app_entry.c
wolfssl-template-project configuration.xml
README.md include.am user_settings.h
Projects
common strings.h unistd.h user_settings.h wolfssl_dummy.c
test
src key_data.c key_data.h test_main.c wolf_client.c wolf_server.c wolfssl_demo.h
tools generate_rsa_keypair.sh genhexbuf.pl rsa_pss_sign.sh
wolfssl
src .gitkeep
wolfcrypt
src .gitkeep
README include.am
RA6M3
benchmark-wolfcrypt
common .gitkeep
script .gitkeep
src wolfssl_thread_entry.c
client-wolfssl
common
src .gitkeep
script .gitkeep
src wolfssl_thread_entry.c
wolfssl_thread_entry.h
common
ra6m3g README.md
src freertos_tcp_port.c
user_settings.h util.h
server-wolfssl
common
src .gitkeep
script .gitkeep
src wolfssl_thread_entry.c
wolfssl_thread_entry.h
test-wolfcrypt
common .gitkeep
script .gitkeep
src wolfssl_thread_entry.c
wolfssl
src .gitkeep
wolfcrypt .gitkeep
README.md README_APRA6M_en.md README_APRA6M_jp.md include.am
RA6M3G README.md
RA6M4
common user_settings.h wolfssl_demo.h
test
key_data key_data.h key_data_sce.c
src
SEGGER_RTT myprint.c
common .gitignore
test_main.c wolf_client.c wolfssl_sce_unit_test.c
test_RA6M4Debug.launch
tools
example_keys generate_SignedCA.sh rsa_private.pem rsa_public.pem
README.md
README.md include.am
RX65N
GR-ROSE
common strings.h unistd.h user_settings.h wolfssl_dummy.c
smc smc.scfg
test
src key_data.c key_data.h test_main.c wolf_client.c wolf_server.c wolfssl_demo.h
test.rcpc test_HardwareDebug.launch
tools
example_keys generate_SignedCA.sh rsa_private.pem rsa_public.pem
README.md
wolfssl wolfssl.rcpc
README_EN.md README_JP.md include.am
RSK
resource section.esi
wolfssl wolfssl.rcpc
wolfssl_demo key_data.c key_data.h user_settings.h wolfssl_demo.c wolfssl_demo.h
InstructionManualForExample_RSK+RX65N-2MB_EN.pdf InstructionManualForExample_RSK+RX65N-2MB_JP.pdf README_EN.md README_JP.md include.am
RX72N
EnvisionKit
Simple
common sectioninfo.esi wolfssl_dummy.c
test
src
client simple_tcp_client.c simple_tls_tsip_client.c
server simple_tcp_server.c simple_tls_server.c
test_main.c wolfssl_simple_demo.h
test.rcpc test.scfg test_HardwareDebug.launch
wolfssl wolfssl.rcpc
README_EN.md README_JP.md
resource section.esi
tools
example_keys generate_SignedCA.sh rsa_private.pem rsa_public.pem
README.md
wolfssl wolfssl.rcpc
wolfssl_demo key_data.c key_data.h user_settings.h wolfssl_demo.c wolfssl_demo.h wolfssl_tsip_unit_test.c
InstructionManualForExample_RX72N_EnvisonKit_EN.pdf InstructionManualForExample_RX72N_EnvisonKit_JP.pdf README_EN.md README_JP.md include.am
RZN2L
common user_settings.h wolfssl_demo.h
test
src
serial_io app_print.c
test wolf_client.c wolf_server.c wolfssl_rsip_unit_test.c
wolfCrypt .gitignore
wolfSSL .gitignore
local_system_init.c rzn2l_tst_thread_entry.c wolfssl_dummy.c
README.md include.am
SK-S7G2
common user_settings.h
wolfssl_lib configuration.xml
.gitignore README.md include.am
STARCORE README.txt include.am starcore_test.c user_settings.h
STM32Cube README.md STM32_Benchmarks.md default_conf.ftl include.am main.c wolfssl_example.c wolfssl_example.h
SimplicityStudio README.md include.am test_wolf.c user_settings.h
TRUESTUDIO
wolfssl user_settings.h
README include.am
VS-ARM README.md include.am user_settings.h wolfssl.sln wolfssl.vcxproj
VS-AZURE-SPHERE
client app_manifest.json client.c client.h client.vcxproj
server app_manifest.json server.c server.h server.vcxproj
shared util.h
wolfcrypt_test app_manifest.json wolfcrypt_test.vcxproj
README.md include.am user_settings.h wolfssl.sln wolfssl.vcxproj
VisualDSP include.am user_settings.h wolf_tasks.c
WICED-STUDIO README include.am user_settings.h
WIN README.txt include.am test.vcxproj user_settings.h user_settings_dtls.h wolfssl-fips.sln wolfssl-fips.vcxproj
WIN-SGX ReadMe.txt include.am wolfSSL_SGX.edl wolfSSL_SGX.sln wolfSSL_SGX.vcxproj
WIN-SRTP-KDF-140-3 README.txt include.am resource.h test.vcxproj user_settings.h wolfssl-fips.rc wolfssl-fips.sln wolfssl-fips.vcxproj
WIN10 README.txt include.am resource.h test.vcxproj user_settings.h wolfssl-fips.rc wolfssl-fips.sln wolfssl-fips.vcxproj
WINCE README.md include.am user_settings.h user_settings.h.140-2-deprecated
WORKBENCH README.md include.am
XCODE
Benchmark
wolfBench
Assets.xcassets
AppIcon.appiconset Contents.json
Base.lproj LaunchScreen.storyboard Main.storyboard
AppDelegate.h AppDelegate.m Info.plist ViewController.h ViewController.m main.m
wolfBench.xcodeproj project.pbxproj
include.am
wolfssl-FIPS.xcodeproj project.pbxproj
wolfssl.xcodeproj project.pbxproj
wolfssl_testsuite.xcodeproj project.pbxproj
README.md build-for-i386.sh include.am user_settings.h
XCODE-FIPSv2
macOS-C++
Intel user_settings.h
M1 user_settings.h
include.am user_settings.h
XCODE-FIPSv5 README include.am user_settings.h
XCODE-FIPSv6 README include.am user_settings.h
XilinxSDK
2018_2 lscript.ld
2019_2
wolfCrypt_example
src lscript.ld
wolfCrypt_example_system wolfCrypt_example_system.sprj
2022_1
wolfCrypt_FreeRTOS_example wolfCrypt_FreeRTOS_example.prj
wolfCrypt_FreeRTOS_example_system wolfCrypt_FreeRTOS_example_system.sprj
wolfCrypt_example wolfCrypt_example.prj
wolfCrypt_example_system wolfCrypt_example_system.sprj
.gitignore
README.md bench.sh combine.sh eclipse_formatter_profile.xml graph.sh include.am user_settings.h wolfssl_example.c
apple-universal
wolfssl-multiplatform
wolfssl-multiplatform
Assets.xcassets
AccentColor.colorset Contents.json
AppIcon.appiconset Contents.json
Contents.json
ContentView.swift simple_client_example.c simple_client_example.h wolfssl-multiplatform-Bridging-Header.h wolfssl_multiplatform.entitlements wolfssl_multiplatformApp.swift wolfssl_test_driver.c wolfssl_test_driver.h
wolfssl-multiplatform.xcodeproj project.pbxproj
.gitignore README.md build-wolfssl-framework.sh include.am
iotsafe Makefile README.md ca-cert.c devices.c devices.h include.am main.c memory-tls.c startup.c target.ld user_settings.h
iotsafe-raspberrypi Makefile README.md client-tls13.c include.am main.c
mynewt README.md apps.wolfcrypttest.pkg.yml crypto.wolfssl.pkg.yml crypto.wolfssl.syscfg.yml include.am setup.sh
zephyr README.md include.am
include.am
RTOS
nuttx
wolfssl .gitignore Kconfig Make.defs Makefile README.md setup-wolfssl.sh user_settings.h
include.am
bsdkm Makefile README.md bsdkm_wc_port.h include.am wolfkmod.c wolfkmod_aes.c x86_vecreg.c
certs
1024 ca-cert.der ca-cert.pem ca-key.der ca-key.pem client-cert.der client-cert.pem client-key.der client-key.pem client-keyPub.der dh1024.der dh1024.pem dsa-pub-1024.pem dsa1024.der dsa1024.pem include.am rsa1024.der server-cert.der server-cert.pem server-key.der server-key.pem
3072 client-cert.der client-cert.pem client-key.der client-key.pem client-keyPub.der include.am
4096 client-cert.der client-cert.pem client-key.der client-key.pem client-keyPub.der include.am
acert
rsa_pss acert.pem acert_ietf.pem acert_ietf_pubkey.pem acert_pubkey.pem
acert.pem acert_ietf.pem acert_ietf_pubkey.pem acert_pubkey.pem include.am
aia ca-issuers-cert.pem multi-aia-cert.pem overflow-aia-cert.pem
crl
extra-crls ca-int-cert-revoked.pem claim-root.pem crl_critical_entry.pem crlnum_57oct.pem crlnum_64oct.pem general-server-crl.pem large_crlnum.pem large_crlnum2.pem
hash_der 0fdb2da4.r0
hash_pem 0fdb2da4.r0
bad_time_fmt.pem ca-int-ecc.pem ca-int.pem ca-int2-ecc.pem ca-int2.pem caEcc384Crl.pem caEccCrl.der caEccCrl.pem cliCrl.pem client-int-ecc.pem client-int.pem crl.der crl.pem crl.revoked crl2.der crl2.pem crl_reason.pem crl_rsapss.pem eccCliCRL.pem eccSrvCRL.pem gencrls.sh include.am server-goodaltCrl.pem server-goodaltwildCrl.pem server-goodcnCrl.pem server-goodcnwildCrl.pem server-int-ecc.pem server-int.pem wolfssl.cnf
dilithium bench_dilithium_level2_key.der bench_dilithium_level3_key.der bench_dilithium_level5_key.der include.am
ecc bp256r1-key.der bp256r1-key.pem ca-secp256k1-cert.pem ca-secp256k1-key.pem client-bp256r1-cert.der client-bp256r1-cert.pem client-secp256k1-cert.der client-secp256k1-cert.pem genecc.sh include.am secp256k1-key.der secp256k1-key.pem secp256k1-param.pem secp256k1-privkey.der secp256k1-privkey.pem server-bp256r1-cert.der server-bp256r1-cert.pem server-secp256k1-cert.der server-secp256k1-cert.pem server2-secp256k1-cert.der server2-secp256k1-cert.pem wolfssl.cnf wolfssl_384.cnf
ed25519 ca-ed25519-key.der ca-ed25519-key.pem ca-ed25519-priv.der ca-ed25519-priv.pem ca-ed25519.der ca-ed25519.pem client-ed25519-key.der client-ed25519-key.pem client-ed25519-priv.der client-ed25519-priv.pem client-ed25519.der client-ed25519.pem eddsa-ed25519.der eddsa-ed25519.pem gen-ed25519-certs.sh gen-ed25519-keys.sh gen-ed25519.sh include.am root-ed25519-key.der root-ed25519-key.pem root-ed25519-priv.der root-ed25519-priv.pem root-ed25519.der root-ed25519.pem server-ed25519-cert.pem server-ed25519-key.der server-ed25519-key.pem server-ed25519-priv.der server-ed25519-priv.pem server-ed25519.der server-ed25519.pem
ed448 ca-ed448-key.der ca-ed448-key.pem ca-ed448-priv.der ca-ed448-priv.pem ca-ed448.der ca-ed448.pem client-ed448-key.der client-ed448-key.pem client-ed448-priv.der client-ed448-priv.pem client-ed448.der client-ed448.pem gen-ed448-certs.sh gen-ed448-keys.sh include.am root-ed448-key.der root-ed448-key.pem root-ed448-priv.der root-ed448-priv.pem root-ed448.der root-ed448.pem server-ed448-cert.pem server-ed448-key.der server-ed448-key.pem server-ed448-priv.der server-ed448-priv.pem server-ed448.der server-ed448.pem
external DigiCertGlobalRootCA.pem README.txt ca-digicert-ev.pem ca-globalsign-root.pem ca-google-root.pem ca_collection.pem include.am
falcon bench_falcon_level1_key.der bench_falcon_level5_key.der include.am
intermediate
ca_false_intermediate gentestcert.sh int_ca.key server.key test_ca.key test_ca.pem test_int_not_cacert.pem test_sign_bynoca_srv.pem wolfssl_base.conf wolfssl_srv.conf
ca-ecc-bad-aki.der ca-ecc-bad-aki.pem ca-int-cert.der ca-int-cert.pem ca-int-ecc-cert.der ca-int-ecc-cert.pem ca-int-ecc-key.der ca-int-ecc-key.pem ca-int-key.der ca-int-key.pem ca-int2-cert.der ca-int2-cert.pem ca-int2-ecc-cert.der ca-int2-ecc-cert.pem ca-int2-ecc-key.der ca-int2-ecc-key.pem ca-int2-key.der ca-int2-key.pem client-chain-alt-ecc.pem client-chain-alt.pem client-chain-ecc.der client-chain-ecc.pem client-chain.der client-chain.pem client-int-cert.der client-int-cert.pem client-int-ecc-cert.der client-int-ecc-cert.pem genintcerts.sh include.am server-chain-alt-ecc.pem server-chain-alt.pem server-chain-ecc.der server-chain-ecc.pem server-chain-short.pem server-chain.der server-chain.pem server-int-cert.der server-int-cert.pem server-int-ecc-cert.der server-int-ecc-cert.pem
lms bc_hss_L2_H5_W8_root.der bc_hss_L3_H5_W4_root.der bc_lms_chain_ca.der bc_lms_chain_leaf.der bc_lms_native_bc_root.der bc_lms_sha256_h10_w8_root.der bc_lms_sha256_h5_w4_root.der include.am
mldsa README.txt include.am mldsa44-cert.der mldsa44-cert.pem mldsa44-key.pem mldsa44_bare-priv.der mldsa44_bare-seed.der mldsa44_oqskeypair.der mldsa44_priv-only.der mldsa44_pub-spki.der mldsa44_seed-only.der mldsa44_seed-priv.der mldsa65-cert.der mldsa65-cert.pem mldsa65-key.pem mldsa65_bare-priv.der mldsa65_bare-seed.der mldsa65_oqskeypair.der mldsa65_priv-only.der mldsa65_pub-spki.der mldsa65_seed-only.der mldsa65_seed-priv.der mldsa87-cert.der mldsa87-cert.pem mldsa87-key.pem mldsa87_bare-priv.der mldsa87_bare-seed.der mldsa87_oqskeypair.der mldsa87_priv-only.der mldsa87_pub-spki.der mldsa87_seed-only.der mldsa87_seed-priv.der
ocsp imposter-root-ca-cert.der imposter-root-ca-cert.pem imposter-root-ca-key.der imposter-root-ca-key.pem include.am index-ca-and-intermediate-cas.txt index-ca-and-intermediate-cas.txt.attr index-intermediate1-ca-issued-certs.txt index-intermediate1-ca-issued-certs.txt.attr index-intermediate2-ca-issued-certs.txt index-intermediate2-ca-issued-certs.txt.attr index-intermediate3-ca-issued-certs.txt index-intermediate3-ca-issued-certs.txt.attr intermediate1-ca-cert.der intermediate1-ca-cert.pem intermediate1-ca-key.der intermediate1-ca-key.pem intermediate2-ca-cert.der intermediate2-ca-cert.pem intermediate2-ca-key.der intermediate2-ca-key.pem intermediate3-ca-cert.der intermediate3-ca-cert.pem intermediate3-ca-key.der intermediate3-ca-key.pem ocsp-responder-cert.der ocsp-responder-cert.pem ocsp-responder-key.der ocsp-responder-key.pem openssl.cnf renewcerts-for-test.sh renewcerts.sh root-ca-cert.der root-ca-cert.pem root-ca-crl.pem root-ca-key.der root-ca-key.pem server1-cert.der server1-cert.pem server1-chain-noroot.pem server1-key.der server1-key.pem server2-cert.der server2-cert.pem server2-key.der server2-key.pem server3-cert.der server3-cert.pem server3-key.der server3-key.pem server4-cert.der server4-cert.pem server4-key.der server4-key.pem server5-cert.der server5-cert.pem server5-key.der server5-key.pem test-leaf-response.der test-multi-response.der test-response-nointern.der test-response-rsapss.der test-response.der
p521 ca-p521-key.der ca-p521-key.pem ca-p521-priv.der ca-p521-priv.pem ca-p521.der ca-p521.pem client-p521-key.der client-p521-key.pem client-p521-priv.der client-p521-priv.pem client-p521.der client-p521.pem gen-p521-certs.sh gen-p521-keys.sh include.am root-p521-key.der root-p521-key.pem root-p521-priv.der root-p521-priv.pem root-p521.der root-p521.pem server-p521-cert.pem server-p521-key.der server-p521-key.pem server-p521-priv.der server-p521-priv.pem server-p521.der server-p521.pem
renewcerts wolfssl.cnf
rpk client-cert-rpk.der client-ecc-cert-rpk.der include.am server-cert-rpk.der server-ecc-cert-rpk.der
rsapss ca-3072-rsapss-key.der ca-3072-rsapss-key.pem ca-3072-rsapss-priv.der ca-3072-rsapss-priv.pem ca-3072-rsapss.der ca-3072-rsapss.pem ca-rsapss-key.der ca-rsapss-key.pem ca-rsapss-priv.der ca-rsapss-priv.pem ca-rsapss.der ca-rsapss.pem client-3072-rsapss-key.der client-3072-rsapss-key.pem client-3072-rsapss-priv.der client-3072-rsapss-priv.pem client-3072-rsapss.der client-3072-rsapss.pem client-rsapss-key.der client-rsapss-key.pem client-rsapss-priv.der client-rsapss-priv.pem client-rsapss.der client-rsapss.pem gen-rsapss-keys.sh include.am renew-rsapss-certs.sh root-3072-rsapss-key.der root-3072-rsapss-key.pem root-3072-rsapss-priv.der root-3072-rsapss-priv.pem root-3072-rsapss.der root-3072-rsapss.pem root-rsapss-key.der root-rsapss-key.pem root-rsapss-priv.der root-rsapss-priv.pem root-rsapss.der root-rsapss.pem server-3072-rsapss-cert.pem server-3072-rsapss-key.der server-3072-rsapss-key.pem server-3072-rsapss-priv.der server-3072-rsapss-priv.pem server-3072-rsapss.der server-3072-rsapss.pem server-mix-rsapss-cert.pem server-rsapss-cert.pem server-rsapss-key.der server-rsapss-key.pem server-rsapss-priv.der server-rsapss-priv.pem server-rsapss.der server-rsapss.pem
sia timestamping-sia-cert.pem
slhdsa bench_slhdsa_sha2_128f_key.der bench_slhdsa_sha2_128s_key.der bench_slhdsa_sha2_192f_key.der bench_slhdsa_sha2_192s_key.der bench_slhdsa_sha2_256f_key.der bench_slhdsa_sha2_256s_key.der bench_slhdsa_shake128f_key.der bench_slhdsa_shake128s_key.der bench_slhdsa_shake192f_key.der bench_slhdsa_shake192s_key.der bench_slhdsa_shake256f_key.der bench_slhdsa_shake256s_key.der client-mldsa44-priv.pem client-mldsa44-sha2.der client-mldsa44-sha2.pem client-mldsa44-shake.der client-mldsa44-shake.pem gen-slhdsa-mldsa-certs.sh include.am root-slhdsa-sha2-128s-priv.der root-slhdsa-sha2-128s-priv.pem root-slhdsa-sha2-128s.der root-slhdsa-sha2-128s.pem root-slhdsa-shake-128s-priv.der root-slhdsa-shake-128s-priv.pem root-slhdsa-shake-128s.der root-slhdsa-shake-128s.pem server-mldsa44-priv.pem server-mldsa44-sha2.der server-mldsa44-sha2.pem server-mldsa44-shake.der server-mldsa44-shake.pem
sm2 ca-sm2-key.der ca-sm2-key.pem ca-sm2-priv.der ca-sm2-priv.pem ca-sm2.der ca-sm2.pem client-sm2-key.der client-sm2-key.pem client-sm2-priv.der client-sm2-priv.pem client-sm2.der client-sm2.pem fix_sm2_spki.py gen-sm2-certs.sh gen-sm2-keys.sh include.am root-sm2-key.der root-sm2-key.pem root-sm2-priv.der root-sm2-priv.pem root-sm2.der root-sm2.pem self-sm2-cert.pem self-sm2-key.pem self-sm2-priv.pem server-sm2-cert.der server-sm2-cert.pem server-sm2-key.der server-sm2-key.pem server-sm2-priv.der server-sm2-priv.pem server-sm2.der server-sm2.pem
statickeys dh-ffdhe2048-params.pem dh-ffdhe2048-pub.der dh-ffdhe2048-pub.pem dh-ffdhe2048.der dh-ffdhe2048.pem ecc-secp256r1.der ecc-secp256r1.pem gen-static.sh include.am x25519-pub.der x25519-pub.pem x25519.der x25519.pem
test
expired expired-ca.der expired-ca.pem expired-cert.der expired-cert.pem
catalog.txt cert-bad-neg-int.der cert-bad-oid.der cert-bad-utf8.der cert-ext-ia.cfg cert-ext-ia.der cert-ext-ia.pem cert-ext-joi.cfg cert-ext-joi.der cert-ext-joi.pem cert-ext-mnc.der cert-ext-multiple.cfg cert-ext-multiple.der cert-ext-multiple.pem cert-ext-nc-combined.der cert-ext-nc-combined.pem cert-ext-nc.cfg cert-ext-nc.der cert-ext-nc.pem cert-ext-ncdns.der cert-ext-ncdns.pem cert-ext-ncip.der cert-ext-ncip.pem cert-ext-ncmixed.der cert-ext-ncmulti.der cert-ext-ncmulti.pem cert-ext-ncrid.der cert-ext-ncrid.pem cert-ext-nct.cfg cert-ext-nct.der cert-ext-nct.pem cert-ext-ndir-exc.cfg cert-ext-ndir-exc.der cert-ext-ndir-exc.pem cert-ext-ndir.cfg cert-ext-ndir.der cert-ext-ndir.pem cert-ext-ns.der cert-over-max-altnames.cfg cert-over-max-altnames.der cert-over-max-altnames.pem cert-over-max-nc.cfg cert-over-max-nc.der cert-over-max-nc.pem client-ecc-cert-ski.hex cn-ip-literal.der cn-ip-wildcard.der crit-cert.pem crit-key.pem dh1024.der dh1024.pem dh512.der dh512.pem digsigku.pem encrypteddata.msg gen-badsig.sh gen-ext-certs.sh gen-testcerts.sh include.am kari-keyid-cms.msg ktri-keyid-cms.msg ossl-trusted-cert.pem server-badaltname.der server-badaltname.pem server-badaltnull.der server-badaltnull.pem server-badcn.der server-badcn.pem server-badcnnull.der server-badcnnull.pem server-cert-ecc-badsig.der server-cert-ecc-badsig.pem server-cert-rsa-badsig.der server-cert-rsa-badsig.pem server-duplicate-policy.pem server-garbage.der server-garbage.pem server-goodalt.der server-goodalt.pem server-goodaltwild.der server-goodaltwild.pem server-goodcn.der server-goodcn.pem server-goodcnwild.der server-goodcnwild.pem server-localhost.der server-localhost.pem smime-test-canon.p7s smime-test-multipart-badsig.p7s smime-test-multipart.p7s smime-test.p7s
test-pathlen assemble-chains.sh chainA-ICA1-key.pem chainA-ICA1-pathlen0.pem chainA-assembled.pem chainA-entity-key.pem chainA-entity.pem chainB-ICA1-key.pem chainB-ICA1-pathlen0.pem chainB-ICA2-key.pem chainB-ICA2-pathlen1.pem chainB-assembled.pem chainB-entity-key.pem chainB-entity.pem chainC-ICA1-key.pem chainC-ICA1-pathlen1.pem chainC-assembled.pem chainC-entity-key.pem chainC-entity.pem chainD-ICA1-key.pem chainD-ICA1-pathlen127.pem chainD-assembled.pem chainD-entity-key.pem chainD-entity.pem chainE-ICA1-key.pem chainE-ICA1-pathlen128.pem chainE-assembled.pem chainE-entity-key.pem chainE-entity.pem chainF-ICA1-key.pem chainF-ICA1-pathlen1.pem chainF-ICA2-key.pem chainF-ICA2-pathlen0.pem chainF-assembled.pem chainF-entity-key.pem chainF-entity.pem chainG-ICA1-key.pem chainG-ICA1-pathlen0.pem chainG-ICA2-key.pem chainG-ICA2-pathlen1.pem chainG-ICA3-key.pem chainG-ICA3-pathlen99.pem chainG-ICA4-key.pem chainG-ICA4-pathlen5.pem chainG-ICA5-key.pem chainG-ICA5-pathlen20.pem chainG-ICA6-key.pem chainG-ICA6-pathlen10.pem chainG-ICA7-key.pem chainG-ICA7-pathlen100.pem chainG-assembled.pem chainG-entity-key.pem chainG-entity.pem chainH-ICA1-key.pem chainH-ICA1-pathlen0.pem chainH-ICA2-key.pem chainH-ICA2-pathlen2.pem chainH-ICA3-key.pem chainH-ICA3-pathlen2.pem chainH-ICA4-key.pem chainH-ICA4-pathlen2.pem chainH-assembled.pem chainH-entity-key.pem chainH-entity.pem chainI-ICA1-key.pem chainI-ICA1-no_pathlen.pem chainI-ICA2-key.pem chainI-ICA2-no_pathlen.pem chainI-ICA3-key.pem chainI-ICA3-pathlen2.pem chainI-assembled.pem chainI-entity-key.pem chainI-entity.pem chainJ-ICA1-key.pem chainJ-ICA1-no_pathlen.pem chainJ-ICA2-key.pem chainJ-ICA2-no_pathlen.pem chainJ-ICA3-key.pem chainJ-ICA3-no_pathlen.pem chainJ-ICA4-key.pem chainJ-ICA4-pathlen2.pem chainJ-assembled.pem chainJ-entity-key.pem chainJ-entity.pem include.am refreshkeys.sh
test-serial0 ee_normal.pem ee_serial0.pem generate_certs.sh include.am intermediate_serial0.pem root_serial0.pem root_serial0_key.pem selfsigned_nonca_serial0.pem
xmss bc_xmss_chain_ca.der bc_xmss_chain_leaf.der bc_xmss_sha2_10_256_root.der bc_xmss_sha2_16_256_root.der bc_xmssmt_sha2_20_2_256_root.der bc_xmssmt_sha2_20_4_256_root.der bc_xmssmt_sha2_40_8_256_root.der include.am
ca-cert-chain.der ca-cert.der ca-cert.pem ca-ecc-cert.der ca-ecc-cert.pem ca-ecc-key.der ca-ecc-key.pem ca-ecc384-cert.der ca-ecc384-cert.pem ca-ecc384-key.der ca-ecc384-key.pem ca-key-pkcs8-attribute.der ca-key.der ca-key.pem check_dates.sh client-absolute-urn.pem client-ca-cert.der client-ca-cert.pem client-ca.pem client-cert-ext.der client-cert-ext.pem client-cert.der client-cert.pem client-crl-dist.der client-crl-dist.pem client-ecc-ca-cert.der client-ecc-ca-cert.pem client-ecc-cert.der client-ecc-cert.pem client-ecc384-cert.der client-ecc384-cert.pem client-ecc384-key.der client-ecc384-key.pem client-key.der client-key.pem client-keyEnc.pem client-keyPub.der client-keyPub.pem client-relative-uri.pem client-uri-cert.pem csr.attr.der csr.dsa.der csr.dsa.pem csr.ext.der csr.signed.der dh-priv-2048.der dh-priv-2048.pem dh-pub-2048.der dh-pub-2048.pem dh-pubkey-2048.der dh2048.der dh2048.pem dh3072.der dh3072.pem dh4096.der dh4096.pem dsa-pubkey-2048.der dsa2048.der dsa2048.pem dsa3072.der dsaparams.der dsaparams.pem ecc-client-key.der ecc-client-key.pem ecc-client-keyPub.der ecc-client-keyPub.pem ecc-key-comp.pem ecc-keyPkcs8.der ecc-keyPkcs8.pem ecc-keyPkcs8Enc.der ecc-keyPkcs8Enc.pem ecc-keyPub.der ecc-keyPub.pem ecc-params.der ecc-params.pem ecc-privOnlyCert.pem ecc-privOnlyKey.pem ecc-privkey.der ecc-privkey.pem ecc-privkeyPkcs8.der ecc-privkeyPkcs8.pem ecc-rsa-server.p12 empty-issuer-cert.pem entity-no-ca-bool-cert.pem entity-no-ca-bool-key.pem fpki-cert.der fpki-certpol-cert.der gen_revoked.sh include.am renewcerts.sh rid-cert.der rsa-pub-2048.pem rsa2048.der rsa3072.der server-cert-chain.der server-cert.der server-cert.pem server-ecc-comp.der server-ecc-comp.pem server-ecc-rsa.der server-ecc-rsa.pem server-ecc-self.der server-ecc-self.pem server-ecc.der server-ecc.pem server-ecc384-cert.der server-ecc384-cert.pem server-ecc384-key.der server-ecc384-key.pem server-key.der server-key.pem server-keyEnc.pem server-keyPkcs8.der server-keyPkcs8.pem server-keyPkcs8Enc.der server-keyPkcs8Enc.pem server-keyPkcs8Enc12.pem server-keyPkcs8Enc2.pem server-keyPub.der server-keyPub.pem server-revoked-cert.pem server-revoked-key.pem taoCert.txt test-ber-exp02-05-2022.p7b test-degenerate.p7b test-multiple-recipients.p7b test-servercert-rc2.p12 test-servercert.p12 test-stream-dec.p7b test-stream-sign.p7b wolfssl-website-ca.pem x942dh2048.der x942dh2048.pem
cmake
consumer CMakeLists.txt README.md main.c
modules FindARIA.cmake FindOQS.cmake
Config.cmake.in README.md config.in functions.cmake include.am options.h.in wolfssl-config-version.cmake.in wolfssl-targets.cmake.in
debian
source format
changelog.in control.in copyright include.am libwolfssl-dev.install libwolfssl.install rules.in
doc
dox_comments
header_files aes.h arc4.h ascon.h asn.h asn_public.h blake2.h bn.h camellia.h chacha.h chacha20_poly1305.h cmac.h coding.h compress.h cryptocb.h curve25519.h curve448.h des3.h dh.h doxygen_groups.h doxygen_pages.h dsa.h ecc.h eccsi.h ed25519.h ed448.h error-crypt.h evp.h hash.h hmac.h iotsafe.h kdf.h logging.h md2.h md4.h md5.h memory.h ocsp.h pem.h pkcs11.h pkcs7.h poly1305.h psa.h puf.h pwdbased.h quic.h random.h ripemd.h rsa.h sakke.h sha.h sha256.h sha3.h sha512.h signature.h siphash.h srp.h ssl.h tfm.h types.h wc_encrypt.h wc_port.h wc_she.h wc_slhdsa.h wolfio.h
header_files-ja aes.h arc4.h ascon.h asn.h asn_public.h blake2.h bn.h camellia.h chacha.h chacha20_poly1305.h cmac.h coding.h compress.h cryptocb.h curve25519.h curve448.h des3.h dh.h doxygen_groups.h doxygen_pages.h dsa.h ecc.h eccsi.h ed25519.h ed448.h error-crypt.h evp.h hash.h hmac.h iotsafe.h kdf.h logging.h md2.h md4.h md5.h memory.h ocsp.h pem.h pkcs11.h pkcs7.h poly1305.h psa.h pwdbased.h quic.h random.h ripemd.h rsa.h sakke.h sha.h sha256.h sha3.h sha512.h signature.h siphash.h srp.h ssl.h tfm.h types.h wc_encrypt.h wc_port.h wolfio.h
formats
html
html_changes
search search.css search.js
customdoxygen.css doxygen.css menu.js menudata.js tabs.css
Doxyfile footer.html header.html mainpage.dox
pdf Doxyfile header.tex
images wolfssl_logo.png
QUIC.md README.txt README_DOXYGEN check_api.sh generate_documentation.sh include.am
examples
asn1 asn1.c dumpasn1.cfg gen_oid_names.rb include.am oid_names.h
async Makefile README.md async_client.c async_server.c async_tls.c async_tls.h include.am user_settings.h
benchmark include.am tls_bench.c tls_bench.h
client client.c client.h client.sln client.vcproj client.vcxproj include.am
configs README.md include.am user_settings_EBSnet.h user_settings_all.h user_settings_arduino.h user_settings_baremetal.h user_settings_ca.h user_settings_curve25519nonblock.h user_settings_dtls13.h user_settings_eccnonblock.h user_settings_espressif.h user_settings_fipsv2.h user_settings_fipsv5.h user_settings_min_ecc.h user_settings_openssl_compat.h user_settings_pkcs7.h user_settings_platformio.h user_settings_pq.h user_settings_rsa_only.h user_settings_stm32.h user_settings_template.h user_settings_tls12.h user_settings_tls13.h user_settings_wolfboot_keytools.h user_settings_wolfssh.h user_settings_wolftpm.h
crypto_policies
default wolfssl.txt
future wolfssl.txt
legacy wolfssl.txt
echoclient echoclient.c echoclient.h echoclient.sln echoclient.vcproj echoclient.vcxproj include.am quit
echoserver echoserver.c echoserver.h echoserver.sln echoserver.vcproj echoserver.vcxproj include.am
ocsp_responder include.am ocsp_responder.c ocsp_responder.h
pem include.am pem.c
sctp include.am sctp-client-dtls.c sctp-client.c sctp-server-dtls.c sctp-server.c
server include.am server.c server.h server.sln server.vcproj server.vcxproj
README.md include.am
linuxkm
patches
5.10.17 WOLFSSL_LINUXKM_HAVE_GET_RANDOM_CALLBACKS-5v10v17.patch
5.10.236 WOLFSSL_LINUXKM_HAVE_GET_RANDOM_CALLBACKS-5v10v236.patch
5.14.0-570.58.1.el9_6 WOLFSSL_LINUXKM_HAVE_GET_RANDOM_CALLBACKS-5v14-570v58v1-el9_6.patch
5.15 WOLFSSL_LINUXKM_HAVE_GET_RANDOM_CALLBACKS-5v15.patch
5.17 WOLFSSL_LINUXKM_HAVE_GET_RANDOM_CALLBACKS-5v17.patch
5.17-ubuntu-jammy-tegra WOLFSSL_LINUXKM_HAVE_GET_RANDOM_CALLBACKS-5v17-ubuntu-jammy-tegra.patch
6.1.73 WOLFSSL_LINUXKM_HAVE_GET_RANDOM_CALLBACKS-6v1v73.patch
6.12 WOLFSSL_LINUXKM_HAVE_GET_RANDOM_CALLBACKS-6v12.patch
6.15 WOLFSSL_LINUXKM_HAVE_GET_RANDOM_CALLBACKS-6v15.patch
7.0 WOLFSSL_LINUXKM_HAVE_GET_RANDOM_CALLBACKS-7v0.patch
regen-patches.sh
Kbuild Makefile README.md get_thread_size.c include.am linuxkm-fips-hash-wrapper.sh linuxkm-fips-hash.c linuxkm_memory.c linuxkm_memory.h linuxkm_wc_port.h lkcapi_aes_glue.c lkcapi_dh_glue.c lkcapi_ecdh_glue.c lkcapi_ecdsa_glue.c lkcapi_glue.c lkcapi_rsa_glue.c lkcapi_sha_glue.c module_exports.c.template module_hooks.c pie_redirect_table.c wolfcrypt.lds x86_vector_register_glue.c
m4 ax_add_am_macro.m4 ax_am_jobserver.m4 ax_am_macros.m4 ax_append_compile_flags.m4 ax_append_flag.m4 ax_append_link_flags.m4 ax_append_to_file.m4 ax_atomic.m4 ax_bsdkm.m4 ax_check_compile_flag.m4 ax_check_link_flag.m4 ax_compiler_version.m4 ax_count_cpus.m4 ax_create_generic_config.m4 ax_debug.m4 ax_file_escapes.m4 ax_harden_compiler_flags.m4 ax_linuxkm.m4 ax_print_to_file.m4 ax_pthread.m4 ax_require_defined.m4 ax_tls.m4 ax_vcs_checkout.m4 hexversion.m4 lib_socket_nsl.m4 visibility.m4
mcapi
wolfcrypt_mcapi.X
nbproject configurations.xml include.am project.xml
Makefile
wolfcrypt_test.X
nbproject configurations.xml include.am project.xml
Makefile
wolfssl.X
nbproject configurations.xml include.am project.xml
Makefile
zlib.X
nbproject configurations.xml include.am project.xml
Makefile
PIC32MZ-serial.h README crypto.c crypto.h include.am mcapi_test.c user_settings.h
mplabx
wolfcrypt_benchmark.X
nbproject configurations.xml include.am project.xml
Makefile
wolfcrypt_test.X
nbproject configurations.xml include.am project.xml
Makefile
wolfssl.X
nbproject configurations.xml include.am project.xml
Makefile
PIC32MZ-serial.h README benchmark_main.c include.am test_main.c user_settings.h
mqx
util_lib
Sources include.am util.c util.h
wolfcrypt_benchmark
Debugger K70FN1M0.mem init_kinetis.tcl mass_erase_kinetis.tcl
Sources include.am main.c main.h
ReferencedRSESystems.xml wolfcrypt_benchmark_twrk70f120m_Int_Flash_DDRData_Debug_PnE_U-MultiLink.launch wolfcrypt_benchmark_twrk70f120m_Int_Flash_DDRData_Release_PnE_U-MultiLink.launch wolfcrypt_benchmark_twrk70f120m_Int_Flash_SramData_Debug_JTrace.jlink wolfcrypt_benchmark_twrk70f120m_Int_Flash_SramData_Debug_JTrace.launch wolfcrypt_benchmark_twrk70f120m_Int_Flash_SramData_Debug_PnE_U-MultiLink.launch wolfcrypt_benchmark_twrk70f120m_Int_Flash_SramData_Release_PnE_U-MultiLink.launch
wolfcrypt_test
Debugger K70FN1M0.mem init_kinetis.tcl mass_erase_kinetis.tcl
Sources include.am main.c main.h
ReferencedRSESystems.xml wolfcrypt_test_twrk70f120m_Int_Flash_DDRData_Debug_PnE_U-MultiLink.launch wolfcrypt_test_twrk70f120m_Int_Flash_DDRData_Release_PnE_U-MultiLink.launch wolfcrypt_test_twrk70f120m_Int_Flash_SramData_Debug_JTrace.jlink wolfcrypt_test_twrk70f120m_Int_Flash_SramData_Debug_JTrace.launch wolfcrypt_test_twrk70f120m_Int_Flash_SramData_Debug_PnE_U-MultiLink.launch wolfcrypt_test_twrk70f120m_Int_Flash_SramData_Release_PnE_U-MultiLink.launch
wolfssl include.am
wolfssl_client
Debugger K70FN1M0.mem init_kinetis.tcl mass_erase_kinetis.tcl
Sources include.am main.c main.h
ReferencedRSESystems.xml wolfssl_client_twrk70f120m_Int_Flash_DDRData_Debug_PnE_U-MultiLink.launch wolfssl_client_twrk70f120m_Int_Flash_DDRData_Release_PnE_U-MultiLink.launch wolfssl_client_twrk70f120m_Int_Flash_SramData_Debug_JTrace.jlink wolfssl_client_twrk70f120m_Int_Flash_SramData_Debug_JTrace.launch wolfssl_client_twrk70f120m_Int_Flash_SramData_Debug_PnE_U-MultiLink.launch wolfssl_client_twrk70f120m_Int_Flash_SramData_Release_PnE_U-MultiLink.launch
README
rpm include.am spec.in
scripts
bench bench_functions.sh
aria-cmake-build-test.sh asn1_oid_sum.pl benchmark.test benchmark_compare.sh cleanup_testfiles.sh crl-gen-openssl.test crl-revoked.test dertoc.pl dtls.test dtlscid.test external.test google.test include.am makedistsmall.sh memtest.sh ocsp-responder-openssl-interop.test ocsp-stapling-with-ca-as-responder.test ocsp-stapling-with-wolfssl-responder.test ocsp-stapling.test ocsp-stapling2.test ocsp-stapling_tls13multi.test ocsp.test openssl.test openssl_srtp.test pem.test ping.test pkcallbacks.test psk.test resume.test rsapss.test sniffer-gen.sh sniffer-ipv6.pcap sniffer-static-rsa.pcap sniffer-testsuite.test sniffer-tls12-keylog.out sniffer-tls12-keylog.pcap sniffer-tls12-keylog.sslkeylog sniffer-tls13-dh-resume.pcap sniffer-tls13-dh.pcap sniffer-tls13-ecc-resume.pcap sniffer-tls13-ecc.pcap sniffer-tls13-hrr.pcap sniffer-tls13-keylog.out sniffer-tls13-keylog.pcap sniffer-tls13-keylog.sslkeylog sniffer-tls13-x25519-resume.pcap sniffer-tls13-x25519.pcap stm32l4-v4_0_1_build.sh tls13.test trusted_peer.test unit.test.in user_settings_asm.sh
src bio.c conf.c crl.c dtls.c dtls13.c include.am internal.c keys.c ocsp.c pk.c pk_ec.c pk_rsa.c quic.c sniffer.c ssl.c ssl_api_cert.c ssl_api_crl_ocsp.c ssl_api_pk.c ssl_asn1.c ssl_bn.c ssl_certman.c ssl_crypto.c ssl_ech.c ssl_load.c ssl_misc.c ssl_p7p12.c ssl_sess.c ssl_sk.c tls.c tls13.c wolfio.c x509.c x509_str.c
sslSniffer
sslSnifferTest README_WIN.md include.am snifftest.c sslSniffTest.vcproj sslSniffTest.vcxproj
README.md sslSniffer.vcproj sslSniffer.vcxproj
support gen-debug-trace-error-codes.sh include.am wolfssl.pc.in
tests
api api.h api_decl.h create_ocsp_test_blobs.py include.am test_aes.c test_aes.h test_arc4.c test_arc4.h test_ascon.c test_ascon.h test_ascon_kats.h test_asn.c test_asn.h test_blake2.c test_blake2.h test_camellia.c test_camellia.h test_certman.c test_certman.h test_chacha.c test_chacha.h test_chacha20_poly1305.c test_chacha20_poly1305.h test_cmac.c test_cmac.h test_curve25519.c test_curve25519.h test_curve448.c test_curve448.h test_des3.c test_des3.h test_dh.c test_dh.h test_digest.h test_dsa.c test_dsa.h test_dtls.c test_dtls.h test_ecc.c test_ecc.h test_ed25519.c test_ed25519.h test_ed448.c test_ed448.h test_evp.c test_evp.h test_evp_cipher.c test_evp_cipher.h test_evp_digest.c test_evp_digest.h test_evp_pkey.c test_evp_pkey.h test_hash.c test_hash.h test_hmac.c test_hmac.h test_md2.c test_md2.h test_md4.c test_md4.h test_md5.c test_md5.h test_mldsa.c test_mldsa.h test_mlkem.c test_mlkem.h test_ocsp.c test_ocsp.h test_ocsp_test_blobs.h test_ossl_asn1.c test_ossl_asn1.h test_ossl_bio.c test_ossl_bio.h test_ossl_bn.c test_ossl_bn.h test_ossl_cipher.c test_ossl_cipher.h test_ossl_dgst.c test_ossl_dgst.h test_ossl_dh.c test_ossl_dh.h test_ossl_dsa.c test_ossl_dsa.h test_ossl_ec.c test_ossl_ec.h test_ossl_ecx.c test_ossl_ecx.h test_ossl_mac.c test_ossl_mac.h test_ossl_obj.c test_ossl_obj.h test_ossl_p7p12.c test_ossl_p7p12.h test_ossl_pem.c test_ossl_pem.h test_ossl_rand.c test_ossl_rand.h test_ossl_rsa.c test_ossl_rsa.h test_ossl_sk.c test_ossl_sk.h test_ossl_x509.c test_ossl_x509.h test_ossl_x509_acert.c test_ossl_x509_acert.h test_ossl_x509_crypto.c test_ossl_x509_crypto.h test_ossl_x509_ext.c test_ossl_x509_ext.h test_ossl_x509_info.c test_ossl_x509_info.h test_ossl_x509_io.c test_ossl_x509_io.h test_ossl_x509_lu.c test_ossl_x509_lu.h test_ossl_x509_name.c test_ossl_x509_name.h test_ossl_x509_pk.c test_ossl_x509_pk.h test_ossl_x509_str.c test_ossl_x509_str.h test_ossl_x509_vp.c test_ossl_x509_vp.h test_pkcs12.c test_pkcs12.h test_pkcs7.c test_pkcs7.h test_poly1305.c test_poly1305.h test_random.c test_random.h test_rc2.c test_rc2.h test_ripemd.c test_ripemd.h test_rsa.c test_rsa.h test_sha.c test_sha.h test_sha256.c test_sha256.h test_sha3.c test_sha3.h test_sha512.c test_sha512.h test_she.c test_she.h test_signature.c test_signature.h test_slhdsa.c test_slhdsa.h test_sm2.c test_sm2.h test_sm3.c test_sm3.h test_sm4.c test_sm4.h test_tls.c test_tls.h test_tls13.c test_tls13.h test_tls_ext.c test_tls_ext.h test_wc_encrypt.c test_wc_encrypt.h test_wolfmath.c test_wolfmath.h test_x509.c test_x509.h
emnet
IP IP.h
Makefile emnet_nonblock_test.c emnet_shim.c
freertos-mem-track-repro FreeRTOS.h repro.c run.sh semphr.h task.h user_settings.h
swdev .gitignore Makefile README.md swdev.c swdev.h swdev_loader.c swdev_loader.h user_settings.h
CONF_FILES_README.md NCONF_test.cnf README TXT_DB.txt api.c include.am quic.c srp.c suites.c test-altchains.conf test-chains.conf test-dhprime.conf test-dtls-downgrade.conf test-dtls-fails-cipher.conf test-dtls-fails.conf test-dtls-group.conf test-dtls-mtu.conf test-dtls-reneg-client.conf test-dtls-reneg-server.conf test-dtls-resume.conf test-dtls-sha2.conf test-dtls-srtp-fails.conf test-dtls-srtp.conf test-dtls.conf test-dtls13-cid.conf test-dtls13-downgrade-fails.conf test-dtls13-downgrade.conf test-dtls13-pq-hybrid-extra-frag.conf test-dtls13-pq-hybrid-extra.conf test-dtls13-pq-hybrid-frag.conf test-dtls13-pq-standalone-frag.conf test-dtls13-pq-standalone.conf test-dtls13-psk.conf test-dtls13.conf test-ecc-cust-curves.conf test-ed25519.conf test-ed448.conf test-enckeys.conf test-fails.conf test-maxfrag-dtls.conf test-maxfrag.conf test-p521.conf test-psk-no-id-sha2.conf test-psk-no-id.conf test-psk.conf test-rsapss.conf test-sctp-sha2.conf test-sctp.conf test-sha2.conf test-sig.conf test-sm2.conf test-tls-downgrade.conf test-tls13-down.conf test-tls13-ecc.conf test-tls13-pq-hybrid-extra.conf test-tls13-pq-hybrid.conf test-tls13-pq-standalone.conf test-tls13-psk-certs.conf test-tls13-psk.conf test-tls13-slhdsa-fail.conf test-tls13-slhdsa-sha2.conf test-tls13-slhdsa-shake.conf test-tls13.conf test-trustpeer.conf test.conf unit.c unit.h utils.c utils.h w64wrapper.c
testsuite include.am testsuite.c testsuite.sln testsuite.vcproj testsuite.vcxproj utils.c utils.h
tirtos
packages
ti
net
wolfssl
tests
EK_TM4C1294XL
wolfcrypt
benchmark TM4C1294NC.icf benchmark.cfg main.c package.bld.hide package.xdc
test TM4C1294NC.icf main.c package.bld.hide package.xdc test.cfg
package.bld package.xdc package.xs
.gitignore README include.am products.mak wolfssl.bld wolfssl.mak
wolfcrypt
benchmark README.md benchmark-VS2022.sln benchmark-VS2022.vcxproj benchmark-VS2022.vcxproj.user benchmark.c benchmark.h benchmark.sln benchmark.vcproj benchmark.vcxproj include.am
src
port
Espressif
esp_crt_bundle README.md cacrt_all.pem cacrt_deprecated.pem cacrt_local.pem esp_crt_bundle.c gen_crt_bundle.py pio_install_cryptography.py
README.md esp32_aes.c esp32_mp.c esp32_sha.c esp32_util.c esp_sdk_mem_lib.c esp_sdk_time_lib.c esp_sdk_wifi_lib.c
Renesas README.md renesas_common.c renesas_fspsm_aes.c renesas_fspsm_rsa.c renesas_fspsm_sha.c renesas_fspsm_util.c renesas_rx64_hw_sha.c renesas_rx64_hw_util.c renesas_tsip_aes.c renesas_tsip_rsa.c renesas_tsip_sha.c renesas_tsip_util.c
af_alg afalg_aes.c afalg_hash.c wc_afalg.c
aria aria-crypt.c aria-cryptocb.c
arm armv8-32-aes-asm.S armv8-32-aes-asm_c.c armv8-32-chacha-asm.S armv8-32-chacha-asm_c.c armv8-32-curve25519.S armv8-32-curve25519_c.c armv8-32-mlkem-asm.S armv8-32-mlkem-asm_c.c armv8-32-poly1305-asm.S armv8-32-poly1305-asm_c.c armv8-32-sha256-asm.S armv8-32-sha256-asm_c.c armv8-32-sha3-asm.S armv8-32-sha3-asm_c.c armv8-32-sha512-asm.S armv8-32-sha512-asm_c.c armv8-aes-asm.S armv8-aes-asm_c.c armv8-aes.c armv8-chacha-asm.S armv8-chacha-asm_c.c armv8-curve25519.S armv8-curve25519_c.c armv8-mlkem-asm.S armv8-mlkem-asm_c.c armv8-poly1305-asm.S armv8-poly1305-asm_c.c armv8-sha256-asm.S armv8-sha256-asm_c.c armv8-sha256.c armv8-sha3-asm.S armv8-sha3-asm_c.c armv8-sha512-asm.S armv8-sha512-asm_c.c armv8-sha512.c cryptoCell.c cryptoCellHash.c thumb2-aes-asm.S thumb2-aes-asm_c.c thumb2-chacha-asm.S thumb2-chacha-asm_c.c thumb2-curve25519.S thumb2-curve25519_c.c thumb2-mlkem-asm.S thumb2-mlkem-asm_c.c thumb2-poly1305-asm.S thumb2-poly1305-asm_c.c thumb2-sha256-asm.S thumb2-sha256-asm_c.c thumb2-sha3-asm.S thumb2-sha3-asm_c.c thumb2-sha512-asm.S thumb2-sha512-asm_c.c
atmel README.md atmel.c
autosar README.md cryif.c crypto.c csm.c include.am test.c
caam README.md caam_aes.c caam_doc.pdf caam_driver.c caam_error.c caam_integrity.c caam_qnx.c caam_sha.c wolfcaam_aes.c wolfcaam_cmac.c wolfcaam_ecdsa.c wolfcaam_fsl_nxp.c wolfcaam_hash.c wolfcaam_hmac.c wolfcaam_init.c wolfcaam_qnx.c wolfcaam_rsa.c wolfcaam_seco.c wolfcaam_x25519.c
cavium README.md README_Octeon.md cavium_nitrox.c cavium_octeon_sync.c
cuda README.md aes-cuda.cu
cypress README.md psoc6_crypto.c
devcrypto README.md devcrypto_aes.c devcrypto_ecdsa.c devcrypto_hash.c devcrypto_hmac.c devcrypto_rsa.c devcrypto_x25519.c wc_devcrypto.c
intel README.md quickassist.c quickassist_mem.c quickassist_sync.c
iotsafe iotsafe.c
kcapi README.md kcapi_aes.c kcapi_dh.c kcapi_ecc.c kcapi_hash.c kcapi_hmac.c kcapi_rsa.c
liboqs liboqs.c
maxim README.md max3266x.c maxq10xx.c
mynewt mynewt_port.c
nxp README.md README_SE050.md casper_port.c dcp_port.c hashcrypt_port.c ksdk_port.c se050_port.c
pic32 pic32mz-crypt.c
ppc32 ppc32-sha256-asm.S ppc32-sha256-asm_c.c ppc32-sha256-asm_cr.c
psa README.md psa.c psa_aes.c psa_hash.c psa_pkcbs.c
riscv riscv-64-aes.c riscv-64-chacha.c riscv-64-poly1305.c riscv-64-sha256.c riscv-64-sha3.c riscv-64-sha512.c
rpi_pico README.md pico.c
silabs README.md silabs_aes.c silabs_ecc.c silabs_hash.c silabs_random.c
st README.md STM32MP13.md STM32MP25.md stm32.c stsafe.c
ti ti-aes.c ti-ccm.c ti-des3.c ti-hash.c
tropicsquare README.md tropic01.c
xilinx xil-aesgcm.c xil-sha3.c xil-versal-glue.c xil-versal-trng.c
nrf51.c
ASN_TEMPLATE.md aes.c aes_asm.S aes_asm.asm aes_gcm_asm.S aes_gcm_asm.asm aes_gcm_x86_asm.S aes_xts_asm.S aes_xts_asm.asm arc4.c ascon.c asm.c asn.c asn_orig.c async.c blake2b.c blake2s.c camellia.c chacha.c chacha20_poly1305.c chacha_asm.S chacha_asm.asm cmac.c coding.c compress.c cpuid.c cryptocb.c curve25519.c curve448.c des3.c dh.c dilithium.c dsa.c ecc.c ecc_fp.c eccsi.c ed25519.c ed448.c error.c evp.c evp_pk.c falcon.c fe_448.c fe_low_mem.c fe_operations.c fe_x25519_128.h fe_x25519_asm.S fp_mont_small.i fp_mul_comba_12.i fp_mul_comba_17.i fp_mul_comba_20.i fp_mul_comba_24.i fp_mul_comba_28.i fp_mul_comba_3.i fp_mul_comba_32.i fp_mul_comba_4.i fp_mul_comba_48.i fp_mul_comba_6.i fp_mul_comba_64.i fp_mul_comba_7.i fp_mul_comba_8.i fp_mul_comba_9.i fp_mul_comba_small_set.i fp_sqr_comba_12.i fp_sqr_comba_17.i fp_sqr_comba_20.i fp_sqr_comba_24.i fp_sqr_comba_28.i fp_sqr_comba_3.i fp_sqr_comba_32.i fp_sqr_comba_4.i fp_sqr_comba_48.i fp_sqr_comba_6.i fp_sqr_comba_64.i fp_sqr_comba_7.i fp_sqr_comba_8.i fp_sqr_comba_9.i fp_sqr_comba_small_set.i ge_448.c ge_low_mem.c ge_operations.c hash.c hmac.c hpke.c include.am integer.c kdf.c logging.c md2.c md4.c md5.c memory.c misc.c pkcs12.c pkcs7.c poly1305.c poly1305_asm.S poly1305_asm.asm puf.c pwdbased.c random.c rc2.c ripemd.c rng_bank.c rsa.c sakke.c sha.c sha256.c sha256_asm.S sha3.c sha3_asm.S sha512.c sha512_asm.S signature.c siphash.c sm2.c sm3.c sm3_asm.S sm4.c sp_arm32.c sp_arm64.c sp_armthumb.c sp_c32.c sp_c64.c sp_cortexm.c sp_dsp32.c sp_int.c sp_sm2_arm32.c sp_sm2_arm64.c sp_sm2_armthumb.c sp_sm2_c32.c sp_sm2_c64.c sp_sm2_cortexm.c sp_sm2_x86_64.c sp_sm2_x86_64_asm.S sp_x86_64.c sp_x86_64_asm.S sp_x86_64_asm.asm srp.c tfm.c wc_dsp.c wc_encrypt.c wc_lms.c wc_lms_impl.c wc_mldsa_asm.S wc_mlkem.c wc_mlkem_asm.S wc_mlkem_poly.c wc_pkcs11.c wc_port.c wc_she.c wc_slhdsa.c wc_xmss.c wc_xmss_impl.c wolfentropy.c wolfevent.c wolfmath.c
test README.md include.am test-VS2022.sln test-VS2022.vcxproj test-VS2022.vcxproj.user test.c test.h test.sln test.vcproj test_paths.h.in
wolfssl
openssl aes.h asn1.h asn1t.h bio.h bn.h buffer.h camellia.h cmac.h cms.h compat_types.h conf.h crypto.h des.h dh.h dsa.h ec.h ec25519.h ec448.h ecdh.h ecdsa.h ed25519.h ed448.h engine.h err.h evp.h fips_rand.h hmac.h include.am kdf.h lhash.h md4.h md5.h modes.h obj_mac.h objects.h ocsp.h opensslconf.h opensslv.h ossl_typ.h pem.h pkcs12.h pkcs7.h rand.h rc4.h ripemd.h rsa.h safestack.h sha.h sha3.h srp.h ssl.h ssl23.h stack.h tls1.h txt_db.h ui.h x509.h x509_vfy.h x509v3.h
wolfcrypt
port
Espressif esp-sdk-lib.h esp32-crypt.h esp_crt_bundle.h
Renesas renesas-fspsm-crypt.h renesas-fspsm-types.h renesas-rx64-hw-crypt.h renesas-tsip-crypt.h renesas_cmn.h renesas_fspsm_internal.h renesas_sync.h renesas_tsip_internal.h renesas_tsip_types.h
af_alg afalg_hash.h wc_afalg.h
aria aria-crypt.h aria-cryptocb.h
arm cryptoCell.h
atmel atmel.h
autosar CryIf.h Crypto.h Csm.h StandardTypes.h
caam caam_driver.h caam_error.h caam_qnx.h wolfcaam.h wolfcaam_aes.h wolfcaam_cmac.h wolfcaam_ecdsa.h wolfcaam_fsl_nxp.h wolfcaam_hash.h wolfcaam_qnx.h wolfcaam_rsa.h wolfcaam_seco.h wolfcaam_sha.h wolfcaam_x25519.h
cavium cavium_nitrox.h cavium_octeon_sync.h
cypress psoc6_crypto.h
devcrypto wc_devcrypto.h
intel quickassist.h quickassist_mem.h quickassist_sync.h
iotsafe iotsafe.h
kcapi kcapi_dh.h kcapi_ecc.h kcapi_hash.h kcapi_hmac.h kcapi_rsa.h wc_kcapi.h
liboqs liboqs.h
maxim max3266x-cryptocb.h max3266x.h maxq10xx.h
nxp casper_port.h dcp_port.h hashcrypt_port.h ksdk_port.h se050_port.h
pic32 pic32mz-crypt.h
psa psa.h
riscv riscv-64-asm.h
rpi_pico pico.h
silabs silabs_aes.h silabs_ecc.h silabs_hash.h silabs_random.h
st stm32.h stsafe.h
ti ti-ccm.h ti-hash.h
tropicsquare tropic01.h
xilinx xil-sha3.h xil-versal-glue.h xil-versal-trng.h
nrf51.h
aes.h arc4.h ascon.h asn.h asn_public.h async.h blake2-impl.h blake2-int.h blake2.h camellia.h chacha.h chacha20_poly1305.h cmac.h coding.h compress.h cpuid.h cryptocb.h curve25519.h curve448.h des3.h dh.h dilithium.h dsa.h ecc.h eccsi.h ed25519.h ed448.h error-crypt.h falcon.h fe_448.h fe_operations.h fips_test.h ge_448.h ge_operations.h hash.h hmac.h hpke.h include.am integer.h kdf.h libwolfssl_sources.h libwolfssl_sources_asm.h logging.h md2.h md4.h md5.h mem_track.h memory.h misc.h mpi_class.h mpi_superclass.h oid_sum.h pkcs11.h pkcs12.h pkcs7.h poly1305.h puf.h pwdbased.h random.h rc2.h ripemd.h rng_bank.h rsa.h sakke.h selftest.h settings.h sha.h sha256.h sha3.h sha512.h signature.h siphash.h sm2.h sm3.h sm4.h sp.h sp_int.h srp.h tfm.h types.h visibility.h wc_encrypt.h wc_lms.h wc_mlkem.h wc_pkcs11.h wc_port.h wc_she.h wc_slhdsa.h wc_xmss.h wolfentropy.h wolfevent.h wolfmath.h
callbacks.h certs_test.h certs_test_sm.h crl.h error-ssl.h include.am internal.h ocsp.h options.h.in quic.h sniffer.h sniffer_error.h sniffer_error.rc ssl.h test.h version.h version.h.in wolfio.h
wrapper
Ada
examples
src aes_verify_main.adb rsa_verify_main.adb sha256_main.adb spark_sockets.adb spark_sockets.ads spark_terminal.adb spark_terminal.ads tls_client.adb tls_client.ads tls_client_main.adb tls_server.adb tls_server.ads tls_server_main.adb
.gitignore alire.toml examples.gpr
tests
src
support test_support.adb test_support.ads tests_root_suite.adb tests_root_suite.ads
aes_bindings_tests.adb aes_bindings_tests.ads rsa_verify_bindings_tests.adb rsa_verify_bindings_tests.ads sha256_bindings_tests.adb sha256_bindings_tests.ads tests.adb
.gitignore README.md alire.toml tests.gpr valgrind.supp
.gitignore README.md ada_binding.c alire.toml default.gpr include.am restricted.adc user_settings.h wolfssl-full_runtime.adb wolfssl-full_runtime.ads wolfssl.adb wolfssl.ads wolfssl.gpr
CSharp
wolfCrypt-Test
Properties AssemblyInfo.cs
App.config wolfCrypt-Test.cs wolfCrypt-Test.csproj
wolfSSL-DTLS-PSK-Server
Properties AssemblyInfo.cs
App.config wolfSSL-DTLS-PSK-Server.cs wolfSSL-DTLS-PSK-Server.csproj
wolfSSL-DTLS-Server
Properties AssemblyInfo.cs
App.config wolfSSL-DTLS-Server.cs wolfSSL-DTLS-Server.csproj
wolfSSL-Example-IOCallbacks
Properties AssemblyInfo.cs
App.config wolfSSL-Example-IOCallbacks.cs wolfSSL-Example-IOCallbacks.csproj
wolfSSL-TLS-Client
Properties AssemblyInfo.cs
App.config wolfSSL-TLS-Client.cs wolfSSL-TLS-Client.csproj
wolfSSL-TLS-PSK-Client
Properties AssemblyInfo.cs
App.config wolfSSL-TLS-PSK-Client.cs wolfSSL-TLS-PSK-Client.csproj
wolfSSL-TLS-PSK-Server
Properties AssemblyInfo.cs
App.config wolfSSL-TLS-PSK-Server.cs wolfSSL-TLS-PSK-Server.csproj
wolfSSL-TLS-Server
Properties AssemblyInfo.cs
App.config wolfSSL-TLS-Server.cs wolfSSL-TLS-Server.csproj
wolfSSL-TLS-ServerThreaded
Properties AssemblyInfo.cs
App.config wolfSSL-TLS-ServerThreaded.cs wolfSSL-TLS-ServerThreaded.csproj
wolfSSL_CSharp
Properties AssemblyInfo.cs Resources.Designer.cs Resources.resx
X509.cs wolfCrypt.cs wolfSSL.cs wolfSSL_CSharp.csproj
README.md include.am user_settings.h wolfSSL_CSharp.sln wolfssl.vcxproj
python README.md
rust
wolfssl-wolfcrypt
src aes.rs blake2.rs chacha20_poly1305.rs cmac.rs cmac_mac.rs curve25519.rs dh.rs dilithium.rs ecc.rs ecdsa.rs ed25519.rs ed448.rs fips.rs hkdf.rs hmac.rs hmac_mac.rs kdf.rs lib.rs lms.rs mlkem.rs mlkem_kem.rs pbkdf2_password_hash.rs prf.rs random.rs rsa.rs rsa_pkcs1v15.rs sha.rs sha_digest.rs sys.rs
tests
common mod.rs
test_aes.rs test_blake2.rs test_chacha20_poly1305.rs test_cmac.rs test_cmac_mac.rs test_curve25519.rs test_dh.rs test_dilithium.rs test_ecc.rs test_ecdsa.rs test_ed25519.rs test_ed448.rs test_hkdf.rs test_hmac.rs test_hmac_mac.rs test_kdf.rs test_lms.rs test_mlkem.rs test_mlkem_kem.rs test_pbkdf2_password_hash.rs test_prf.rs test_random.rs test_rsa.rs test_rsa_pkcs1v15.rs test_sha.rs test_sha_digest.rs test_wolfcrypt.rs
CHANGELOG.md Cargo.lock Cargo.toml Makefile README.md build.rs headers.h
Makefile README.md include.am
include.am
zephyr
samples
wolfssl_benchmark
boards native_sim.conf nrf5340dk_nrf5340_cpuapp.conf nrf5340dk_nrf5340_cpuapp_ns.conf
CMakeLists.txt README install_test.sh prj.conf sample.yaml zephyr_legacy.conf zephyr_v4.1.conf
wolfssl_test
boards native_sim.conf nrf5340dk_nrf5340_cpuapp.conf nrf5340dk_nrf5340_cpuapp_ns.conf
CMakeLists.txt README install_test.sh prj-no-malloc.conf prj.conf sample.yaml zephyr_legacy.conf zephyr_v4.1.conf
wolfssl_tls_sock
boards native_sim.conf
src tls_sock.c
CMakeLists.txt README install_sample.sh prj-no-malloc.conf prj.conf sample.yaml zephyr_legacy.conf zephyr_v4.1.conf
wolfssl_tls_thread
boards native_sim.conf nrf5340dk_nrf5340_cpuapp.conf nrf5340dk_nrf5340_cpuapp_ns.conf
src tls_threaded.c
CMakeLists.txt README install_sample.sh prj.conf sample.yaml zephyr_legacy.conf zephyr_v4.1.conf
wolfssl options.h
CMakeLists.txt Kconfig Kconfig.tls-generic README.md include.am module.yml user_settings-no-malloc.h user_settings.h zephyr_init.c
.codespellexcludelines .cyignore .editorconfig .gitignore .wolfssl_known_macro_extras AUTHORS CMakeLists.txt CMakePresets.json CMakeSettings.json COPYING ChangeLog.md INSTALL LICENSING LPCExpresso.cproject LPCExpresso.project Makefile.am README README-async.md README.md SCRIPTS-LIST SECURITY-POLICY.md SECURITY-REPORT-TEMPLATE.md Vagrantfile autogen.sh commit-tests.sh configure.ac fips-check.sh fips-hash.sh gencertbuf.pl input pull_to_vagrant.sh quit resource.h stamp-h.in valgrind-bash.supp valgrind-error.sh wnr-example.conf wolfssl-VS2022.vcxproj wolfssl.rc wolfssl.vcproj wolfssl.vcxproj wolfssl64.sln
.clangd .gitignore DOCS.md Makefile README.md assert.c core.c crypto.c env.c fs.c http.c ini.c json.c log.c luna.h main.c makext.mk path.c process.c request.c sqlite.c stash.c template.c util.c
wolfssl/wolfcrypt/src/aes_gcm_asm.asm raw 
    1; /* aes_gcm_asm.asm */
    2; /*
    3;  * Copyright (C) 2006-2026 wolfSSL Inc.
    4;  *
    5;  * This file is part of wolfSSL.
    6;  *
    7;  * wolfSSL is free software; you can redistribute it and/or modify
    8;  * it under the terms of the GNU General Public License as published by
    9;  * the Free Software Foundation; either version 3 of the License, or
   10;  * (at your option) any later version.
   11;  *
   12;  * wolfSSL is distributed in the hope that it will be useful,
   13;  * but WITHOUT ANY WARRANTY; without even the implied warranty of
   14;  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   15;  * GNU General Public License for more details.
   16;  *
   17;  * You should have received a copy of the GNU General Public License
   18;  * along with this program; if not, write to the Free Software
   19;  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA
   20;  */
   21
   22IF @Version LT 1200
   23; AVX2 instructions not recognized by old versions of MASM
   24IFNDEF NO_AVX2_SUPPORT
   25NO_AVX2_SUPPORT = 1
   26ENDIF
   27; MOVBE instruction not recognized by old versions of MASM
   28IFNDEF NO_MOVBE_SUPPORT
   29NO_MOVBE_SUPPORT = 1
   30ENDIF
   31ENDIF
   32
   33IFNDEF HAVE_INTEL_AVX1
   34HAVE_INTEL_AVX1 = 1
   35ENDIF
   36IFNDEF NO_AVX2_SUPPORT
   37HAVE_INTEL_AVX2 = 1
   38ENDIF
   39
   40IFNDEF _WIN64
   41_WIN64 = 1
   42ENDIF
   43
   44_DATA SEGMENT
   45ALIGN 16
   46L_GCM_generate_m0_aesni_rev8 QWORD \
   47     08090a0b0c0d0e0fh,  0001020304050607h
   48ptr_L_GCM_generate_m0_aesni_rev8 QWORD L_GCM_generate_m0_aesni_rev8
   49_DATA ENDS
   50_DATA SEGMENT
   51ALIGN 16
   52L_GCM_generate_m0_aesni_mod2_128 QWORD \
   53     0000000000000000h, 0e100000000000000h
   54ptr_L_GCM_generate_m0_aesni_mod2_128 QWORD L_GCM_generate_m0_aesni_mod2_128
   55_DATA ENDS
   56_TEXT SEGMENT READONLY PARA
   57GCM_generate_m0_aesni PROC
   58        sub	rsp, 80
   59        movdqu	OWORD PTR [rsp], xmm6
   60        movdqu	OWORD PTR [rsp+16], xmm7
   61        movdqu	OWORD PTR [rsp+32], xmm8
   62        movdqu	OWORD PTR [rsp+48], xmm9
   63        movdqu	OWORD PTR [rsp+64], xmm10
   64        movdqu	xmm9, OWORD PTR L_GCM_generate_m0_aesni_rev8
   65        movdqu	xmm10, OWORD PTR L_GCM_generate_m0_aesni_mod2_128
   66        pxor	xmm8, xmm8
   67        movdqu	xmm0, OWORD PTR [rcx]
   68        movdqu	OWORD PTR [rdx], xmm8
   69        movdqu	xmm8, xmm0
   70        pshufb	xmm0, xmm9
   71        movdqu	xmm5, xmm0
   72        movdqu	xmm4, xmm0
   73        psllq	xmm5, 63
   74        psrlq	xmm4, 1
   75        movdqu	xmm1, xmm5
   76        pslldq	xmm1, 8
   77        psrldq	xmm5, 8
   78        pshufd	xmm1, xmm1, 255
   79        por	xmm4, xmm5
   80        psrad	xmm1, 31
   81        pand	xmm1, xmm10
   82        pxor	xmm1, xmm4
   83        movdqu	xmm5, xmm1
   84        movdqu	xmm4, xmm1
   85        psllq	xmm5, 63
   86        psrlq	xmm4, 1
   87        movdqu	xmm2, xmm5
   88        pslldq	xmm2, 8
   89        psrldq	xmm5, 8
   90        pshufd	xmm2, xmm2, 255
   91        por	xmm4, xmm5
   92        psrad	xmm2, 31
   93        pand	xmm2, xmm10
   94        pxor	xmm2, xmm4
   95        movdqu	xmm5, xmm2
   96        movdqu	xmm4, xmm2
   97        psllq	xmm5, 63
   98        psrlq	xmm4, 1
   99        movdqu	xmm3, xmm5
  100        pslldq	xmm3, 8
  101        psrldq	xmm5, 8
  102        pshufd	xmm3, xmm3, 255
  103        por	xmm4, xmm5
  104        psrad	xmm3, 31
  105        pand	xmm3, xmm10
  106        pxor	xmm3, xmm4
  107        pshufb	xmm3, xmm9
  108        pshufb	xmm2, xmm9
  109        movdqu	xmm8, xmm3
  110        pshufb	xmm1, xmm9
  111        pshufb	xmm0, xmm9
  112        pxor	xmm8, xmm2
  113        movdqu	OWORD PTR [rdx+16], xmm3
  114        movdqu	OWORD PTR [rdx+32], xmm2
  115        movdqu	OWORD PTR [rdx+48], xmm8
  116        movdqu	OWORD PTR [rdx+64], xmm1
  117        movdqu	xmm4, xmm3
  118        movdqu	xmm5, xmm2
  119        movdqu	xmm6, xmm8
  120        pxor	xmm4, xmm1
  121        pxor	xmm5, xmm1
  122        pxor	xmm6, xmm1
  123        movdqu	OWORD PTR [rdx+80], xmm4
  124        movdqu	OWORD PTR [rdx+96], xmm5
  125        movdqu	OWORD PTR [rdx+112], xmm6
  126        movdqu	OWORD PTR [rdx+128], xmm0
  127        pxor	xmm1, xmm0
  128        movdqu	xmm4, xmm3
  129        movdqu	xmm6, xmm2
  130        pxor	xmm4, xmm0
  131        pxor	xmm6, xmm0
  132        movdqu	OWORD PTR [rdx+144], xmm4
  133        movdqu	OWORD PTR [rdx+160], xmm6
  134        pxor	xmm6, xmm3
  135        movdqu	OWORD PTR [rdx+176], xmm6
  136        movdqu	OWORD PTR [rdx+192], xmm1
  137        movdqu	xmm4, xmm3
  138        movdqu	xmm5, xmm2
  139        movdqu	xmm6, xmm8
  140        pxor	xmm4, xmm1
  141        pxor	xmm5, xmm1
  142        pxor	xmm6, xmm1
  143        movdqu	OWORD PTR [rdx+208], xmm4
  144        movdqu	OWORD PTR [rdx+224], xmm5
  145        movdqu	OWORD PTR [rdx+240], xmm6
  146        movdqu	xmm0, OWORD PTR [rdx]
  147        movdqu	xmm1, OWORD PTR [rdx+16]
  148        movdqu	xmm2, OWORD PTR [rdx+32]
  149        movdqu	xmm3, OWORD PTR [rdx+48]
  150        pshufb	xmm0, xmm9
  151        pshufb	xmm1, xmm9
  152        pshufb	xmm2, xmm9
  153        pshufb	xmm3, xmm9
  154        movdqu	xmm4, xmm0
  155        movdqu	xmm5, xmm1
  156        movdqu	xmm6, xmm2
  157        movdqu	xmm7, xmm3
  158        psllq	xmm4, 60
  159        psllq	xmm5, 60
  160        psllq	xmm6, 60
  161        psllq	xmm7, 60
  162        psrlq	xmm0, 4
  163        psrlq	xmm1, 4
  164        psrlq	xmm2, 4
  165        psrlq	xmm3, 4
  166        psrldq	xmm4, 8
  167        psrldq	xmm5, 8
  168        psrldq	xmm6, 8
  169        psrldq	xmm7, 8
  170        por	xmm0, xmm4
  171        por	xmm1, xmm5
  172        por	xmm2, xmm6
  173        por	xmm3, xmm7
  174        vpshufb	xmm0, xmm0, xmm9
  175        vpshufb	xmm1, xmm1, xmm9
  176        vpshufb	xmm2, xmm2, xmm9
  177        vpshufb	xmm3, xmm3, xmm9
  178        movdqu	OWORD PTR [rdx+256], xmm0
  179        movdqu	OWORD PTR [rdx+272], xmm1
  180        movdqu	OWORD PTR [rdx+288], xmm2
  181        movdqu	OWORD PTR [rdx+304], xmm3
  182        movdqu	xmm0, OWORD PTR [rdx+64]
  183        movdqu	xmm1, OWORD PTR [rdx+80]
  184        movdqu	xmm2, OWORD PTR [rdx+96]
  185        movdqu	xmm3, OWORD PTR [rdx+112]
  186        pshufb	xmm0, xmm9
  187        pshufb	xmm1, xmm9
  188        pshufb	xmm2, xmm9
  189        pshufb	xmm3, xmm9
  190        movdqu	xmm4, xmm0
  191        movdqu	xmm5, xmm1
  192        movdqu	xmm6, xmm2
  193        movdqu	xmm7, xmm3
  194        psllq	xmm4, 60
  195        psllq	xmm5, 60
  196        psllq	xmm6, 60
  197        psllq	xmm7, 60
  198        psrlq	xmm0, 4
  199        psrlq	xmm1, 4
  200        psrlq	xmm2, 4
  201        psrlq	xmm3, 4
  202        psrldq	xmm4, 8
  203        psrldq	xmm5, 8
  204        psrldq	xmm6, 8
  205        psrldq	xmm7, 8
  206        por	xmm0, xmm4
  207        por	xmm1, xmm5
  208        por	xmm2, xmm6
  209        por	xmm3, xmm7
  210        vpshufb	xmm0, xmm0, xmm9
  211        vpshufb	xmm1, xmm1, xmm9
  212        vpshufb	xmm2, xmm2, xmm9
  213        vpshufb	xmm3, xmm3, xmm9
  214        movdqu	OWORD PTR [rdx+320], xmm0
  215        movdqu	OWORD PTR [rdx+336], xmm1
  216        movdqu	OWORD PTR [rdx+352], xmm2
  217        movdqu	OWORD PTR [rdx+368], xmm3
  218        movdqu	xmm0, OWORD PTR [rdx+128]
  219        movdqu	xmm1, OWORD PTR [rdx+144]
  220        movdqu	xmm2, OWORD PTR [rdx+160]
  221        movdqu	xmm3, OWORD PTR [rdx+176]
  222        pshufb	xmm0, xmm9
  223        pshufb	xmm1, xmm9
  224        pshufb	xmm2, xmm9
  225        pshufb	xmm3, xmm9
  226        movdqu	xmm4, xmm0
  227        movdqu	xmm5, xmm1
  228        movdqu	xmm6, xmm2
  229        movdqu	xmm7, xmm3
  230        psllq	xmm4, 60
  231        psllq	xmm5, 60
  232        psllq	xmm6, 60
  233        psllq	xmm7, 60
  234        psrlq	xmm0, 4
  235        psrlq	xmm1, 4
  236        psrlq	xmm2, 4
  237        psrlq	xmm3, 4
  238        psrldq	xmm4, 8
  239        psrldq	xmm5, 8
  240        psrldq	xmm6, 8
  241        psrldq	xmm7, 8
  242        por	xmm0, xmm4
  243        por	xmm1, xmm5
  244        por	xmm2, xmm6
  245        por	xmm3, xmm7
  246        vpshufb	xmm0, xmm0, xmm9
  247        vpshufb	xmm1, xmm1, xmm9
  248        vpshufb	xmm2, xmm2, xmm9
  249        vpshufb	xmm3, xmm3, xmm9
  250        movdqu	OWORD PTR [rdx+384], xmm0
  251        movdqu	OWORD PTR [rdx+400], xmm1
  252        movdqu	OWORD PTR [rdx+416], xmm2
  253        movdqu	OWORD PTR [rdx+432], xmm3
  254        movdqu	xmm0, OWORD PTR [rdx+192]
  255        movdqu	xmm1, OWORD PTR [rdx+208]
  256        movdqu	xmm2, OWORD PTR [rdx+224]
  257        movdqu	xmm3, OWORD PTR [rdx+240]
  258        pshufb	xmm0, xmm9
  259        pshufb	xmm1, xmm9
  260        pshufb	xmm2, xmm9
  261        pshufb	xmm3, xmm9
  262        movdqu	xmm4, xmm0
  263        movdqu	xmm5, xmm1
  264        movdqu	xmm6, xmm2
  265        movdqu	xmm7, xmm3
  266        psllq	xmm4, 60
  267        psllq	xmm5, 60
  268        psllq	xmm6, 60
  269        psllq	xmm7, 60
  270        psrlq	xmm0, 4
  271        psrlq	xmm1, 4
  272        psrlq	xmm2, 4
  273        psrlq	xmm3, 4
  274        psrldq	xmm4, 8
  275        psrldq	xmm5, 8
  276        psrldq	xmm6, 8
  277        psrldq	xmm7, 8
  278        por	xmm0, xmm4
  279        por	xmm1, xmm5
  280        por	xmm2, xmm6
  281        por	xmm3, xmm7
  282        vpshufb	xmm0, xmm0, xmm9
  283        vpshufb	xmm1, xmm1, xmm9
  284        vpshufb	xmm2, xmm2, xmm9
  285        vpshufb	xmm3, xmm3, xmm9
  286        movdqu	OWORD PTR [rdx+448], xmm0
  287        movdqu	OWORD PTR [rdx+464], xmm1
  288        movdqu	OWORD PTR [rdx+480], xmm2
  289        movdqu	OWORD PTR [rdx+496], xmm3
  290        movdqu	xmm6, OWORD PTR [rsp]
  291        movdqu	xmm7, OWORD PTR [rsp+16]
  292        movdqu	xmm8, OWORD PTR [rsp+32]
  293        movdqu	xmm9, OWORD PTR [rsp+48]
  294        movdqu	xmm10, OWORD PTR [rsp+64]
  295        add	rsp, 80
  296        ret
  297GCM_generate_m0_aesni ENDP
  298_TEXT ENDS
  299_DATA SEGMENT
  300ALIGN 16
  301L_aes_gcm_one QWORD \
  302     0000000000000000h,  0000000000000001h
  303ptr_L_aes_gcm_one QWORD L_aes_gcm_one
  304_DATA ENDS
  305_DATA SEGMENT
  306ALIGN 16
  307L_aes_gcm_two QWORD \
  308     0000000000000000h,  0000000000000002h
  309ptr_L_aes_gcm_two QWORD L_aes_gcm_two
  310_DATA ENDS
  311_DATA SEGMENT
  312ALIGN 16
  313L_aes_gcm_three QWORD \
  314     0000000000000000h,  0000000000000003h
  315ptr_L_aes_gcm_three QWORD L_aes_gcm_three
  316_DATA ENDS
  317_DATA SEGMENT
  318ALIGN 16
  319L_aes_gcm_four QWORD \
  320     0000000000000000h,  0000000000000004h
  321ptr_L_aes_gcm_four QWORD L_aes_gcm_four
  322_DATA ENDS
  323_DATA SEGMENT
  324ALIGN 16
  325L_aes_gcm_five QWORD \
  326     0000000000000000h,  0000000000000005h
  327ptr_L_aes_gcm_five QWORD L_aes_gcm_five
  328_DATA ENDS
  329_DATA SEGMENT
  330ALIGN 16
  331L_aes_gcm_six QWORD \
  332     0000000000000000h,  0000000000000006h
  333ptr_L_aes_gcm_six QWORD L_aes_gcm_six
  334_DATA ENDS
  335_DATA SEGMENT
  336ALIGN 16
  337L_aes_gcm_seven QWORD \
  338     0000000000000000h,  0000000000000007h
  339ptr_L_aes_gcm_seven QWORD L_aes_gcm_seven
  340_DATA ENDS
  341_DATA SEGMENT
  342ALIGN 16
  343L_aes_gcm_eight QWORD \
  344     0000000000000000h,  0000000000000008h
  345ptr_L_aes_gcm_eight QWORD L_aes_gcm_eight
  346_DATA ENDS
  347_DATA SEGMENT
  348ALIGN 16
  349L_aes_gcm_bswap_epi64 QWORD \
  350     0001020304050607h,  08090a0b0c0d0e0fh
  351ptr_L_aes_gcm_bswap_epi64 QWORD L_aes_gcm_bswap_epi64
  352_DATA ENDS
  353_DATA SEGMENT
  354ALIGN 16
  355L_aes_gcm_bswap_mask QWORD \
  356     08090a0b0c0d0e0fh,  0001020304050607h
  357ptr_L_aes_gcm_bswap_mask QWORD L_aes_gcm_bswap_mask
  358_DATA ENDS
  359_DATA SEGMENT
  360ALIGN 16
  361L_aes_gcm_mod2_128 QWORD \
  362     0000000000000001h, 0c200000000000000h
  363ptr_L_aes_gcm_mod2_128 QWORD L_aes_gcm_mod2_128
  364_DATA ENDS
  365_TEXT SEGMENT READONLY PARA
  366AES_GCM_encrypt_aesni PROC
  367        push	r13
  368        push	rdi
  369        push	rsi
  370        push	r12
  371        push	rbx
  372        push	r14
  373        push	r15
  374        mov	rdi, rcx
  375        mov	rsi, rdx
  376        mov	r12, r8
  377        mov	rax, r9
  378        mov	r8, QWORD PTR [rsp+96]
  379        mov	r9d, DWORD PTR [rsp+104]
  380        mov	r11d, DWORD PTR [rsp+112]
  381        mov	ebx, DWORD PTR [rsp+120]
  382        mov	r14d, DWORD PTR [rsp+128]
  383        mov	r15, QWORD PTR [rsp+136]
  384        mov	r10d, DWORD PTR [rsp+144]
  385        sub	rsp, 320
  386        movdqu	OWORD PTR [rsp+160], xmm6
  387        movdqu	OWORD PTR [rsp+176], xmm7
  388        movdqu	OWORD PTR [rsp+192], xmm8
  389        movdqu	OWORD PTR [rsp+208], xmm9
  390        movdqu	OWORD PTR [rsp+224], xmm10
  391        movdqu	OWORD PTR [rsp+240], xmm11
  392        movdqu	OWORD PTR [rsp+256], xmm12
  393        movdqu	OWORD PTR [rsp+272], xmm13
  394        movdqu	OWORD PTR [rsp+288], xmm14
  395        movdqu	OWORD PTR [rsp+304], xmm15
  396        pxor	xmm4, xmm4
  397        pxor	xmm6, xmm6
  398        cmp	ebx, 12
  399        mov	edx, ebx
  400        jne	L_AES_GCM_encrypt_aesni_iv_not_12
  401        ; # Calculate values when IV is 12 bytes
  402        ; Set counter based on IV
  403        mov	ecx, 16777216
  404        pinsrq	xmm4, QWORD PTR [rax], 0
  405        pinsrd	xmm4, DWORD PTR [rax+8], 2
  406        pinsrd	xmm4, ecx, 3
  407        ; H = Encrypt X(=0) and T = Encrypt counter
  408        movdqa	xmm1, xmm4
  409        movdqa	xmm5, OWORD PTR [r15]
  410        pxor	xmm1, xmm5
  411        movdqa	xmm7, OWORD PTR [r15+16]
  412        aesenc	xmm5, xmm7
  413        aesenc	xmm1, xmm7
  414        movdqa	xmm7, OWORD PTR [r15+32]
  415        aesenc	xmm5, xmm7
  416        aesenc	xmm1, xmm7
  417        movdqa	xmm7, OWORD PTR [r15+48]
  418        aesenc	xmm5, xmm7
  419        aesenc	xmm1, xmm7
  420        movdqa	xmm7, OWORD PTR [r15+64]
  421        aesenc	xmm5, xmm7
  422        aesenc	xmm1, xmm7
  423        movdqa	xmm7, OWORD PTR [r15+80]
  424        aesenc	xmm5, xmm7
  425        aesenc	xmm1, xmm7
  426        movdqa	xmm7, OWORD PTR [r15+96]
  427        aesenc	xmm5, xmm7
  428        aesenc	xmm1, xmm7
  429        movdqa	xmm7, OWORD PTR [r15+112]
  430        aesenc	xmm5, xmm7
  431        aesenc	xmm1, xmm7
  432        movdqa	xmm7, OWORD PTR [r15+128]
  433        aesenc	xmm5, xmm7
  434        aesenc	xmm1, xmm7
  435        movdqa	xmm7, OWORD PTR [r15+144]
  436        aesenc	xmm5, xmm7
  437        aesenc	xmm1, xmm7
  438        cmp	r10d, 11
  439        movdqa	xmm7, OWORD PTR [r15+160]
  440        jl	L_AES_GCM_encrypt_aesni_calc_iv_12_last
  441        aesenc	xmm5, xmm7
  442        aesenc	xmm1, xmm7
  443        movdqa	xmm7, OWORD PTR [r15+176]
  444        aesenc	xmm5, xmm7
  445        aesenc	xmm1, xmm7
  446        cmp	r10d, 13
  447        movdqa	xmm7, OWORD PTR [r15+192]
  448        jl	L_AES_GCM_encrypt_aesni_calc_iv_12_last
  449        aesenc	xmm5, xmm7
  450        aesenc	xmm1, xmm7
  451        movdqa	xmm7, OWORD PTR [r15+208]
  452        aesenc	xmm5, xmm7
  453        aesenc	xmm1, xmm7
  454        movdqa	xmm7, OWORD PTR [r15+224]
  455L_AES_GCM_encrypt_aesni_calc_iv_12_last:
  456        aesenclast	xmm5, xmm7
  457        aesenclast	xmm1, xmm7
  458        pshufb	xmm5, OWORD PTR L_aes_gcm_bswap_mask
  459        movdqu	OWORD PTR [rsp+144], xmm1
  460        jmp	L_AES_GCM_encrypt_aesni_iv_done
  461L_AES_GCM_encrypt_aesni_iv_not_12:
  462        ; Calculate values when IV is not 12 bytes
  463        ; H = Encrypt X(=0)
  464        movdqa	xmm5, OWORD PTR [r15]
  465        aesenc	xmm5, [r15+16]
  466        aesenc	xmm5, [r15+32]
  467        aesenc	xmm5, [r15+48]
  468        aesenc	xmm5, [r15+64]
  469        aesenc	xmm5, [r15+80]
  470        aesenc	xmm5, [r15+96]
  471        aesenc	xmm5, [r15+112]
  472        aesenc	xmm5, [r15+128]
  473        aesenc	xmm5, [r15+144]
  474        cmp	r10d, 11
  475        movdqa	xmm9, OWORD PTR [r15+160]
  476        jl	L_AES_GCM_encrypt_aesni_calc_iv_1_aesenc_avx_last
  477        aesenc	xmm5, xmm9
  478        aesenc	xmm5, [r15+176]
  479        cmp	r10d, 13
  480        movdqa	xmm9, OWORD PTR [r15+192]
  481        jl	L_AES_GCM_encrypt_aesni_calc_iv_1_aesenc_avx_last
  482        aesenc	xmm5, xmm9
  483        aesenc	xmm5, [r15+208]
  484        movdqa	xmm9, OWORD PTR [r15+224]
  485L_AES_GCM_encrypt_aesni_calc_iv_1_aesenc_avx_last:
  486        aesenclast	xmm5, xmm9
  487        pshufb	xmm5, OWORD PTR L_aes_gcm_bswap_mask
  488        ; Calc counter
  489        ; Initialization vector
  490        cmp	edx, 0
  491        mov	rcx, 0
  492        je	L_AES_GCM_encrypt_aesni_calc_iv_done
  493        cmp	edx, 16
  494        jl	L_AES_GCM_encrypt_aesni_calc_iv_lt16
  495        and	edx, 4294967280
  496L_AES_GCM_encrypt_aesni_calc_iv_16_loop:
  497        movdqu	xmm8, OWORD PTR [rax+rcx]
  498        pshufb	xmm8, OWORD PTR L_aes_gcm_bswap_mask
  499        pxor	xmm4, xmm8
  500        pshufd	xmm1, xmm4, 78
  501        pshufd	xmm2, xmm5, 78
  502        movdqa	xmm3, xmm5
  503        movdqa	xmm0, xmm5
  504        pclmulqdq	xmm3, xmm4, 17
  505        pclmulqdq	xmm0, xmm4, 0
  506        pxor	xmm1, xmm4
  507        pxor	xmm2, xmm5
  508        pclmulqdq	xmm1, xmm2, 0
  509        pxor	xmm1, xmm0
  510        pxor	xmm1, xmm3
  511        movdqa	xmm2, xmm1
  512        movdqa	xmm7, xmm0
  513        movdqa	xmm4, xmm3
  514        pslldq	xmm2, 8
  515        psrldq	xmm1, 8
  516        pxor	xmm7, xmm2
  517        pxor	xmm4, xmm1
  518        movdqa	xmm0, xmm7
  519        movdqa	xmm1, xmm4
  520        psrld	xmm0, 31
  521        psrld	xmm1, 31
  522        pslld	xmm7, 1
  523        pslld	xmm4, 1
  524        movdqa	xmm2, xmm0
  525        pslldq	xmm0, 4
  526        psrldq	xmm2, 12
  527        pslldq	xmm1, 4
  528        por	xmm4, xmm2
  529        por	xmm7, xmm0
  530        por	xmm4, xmm1
  531        movdqa	xmm0, xmm7
  532        movdqa	xmm1, xmm7
  533        movdqa	xmm2, xmm7
  534        pslld	xmm0, 31
  535        pslld	xmm1, 30
  536        pslld	xmm2, 25
  537        pxor	xmm0, xmm1
  538        pxor	xmm0, xmm2
  539        movdqa	xmm1, xmm0
  540        psrldq	xmm1, 4
  541        pslldq	xmm0, 12
  542        pxor	xmm7, xmm0
  543        movdqa	xmm2, xmm7
  544        movdqa	xmm3, xmm7
  545        movdqa	xmm0, xmm7
  546        psrld	xmm2, 1
  547        psrld	xmm3, 2
  548        psrld	xmm0, 7
  549        pxor	xmm2, xmm3
  550        pxor	xmm2, xmm0
  551        pxor	xmm2, xmm1
  552        pxor	xmm2, xmm7
  553        pxor	xmm4, xmm2
  554        add	ecx, 16
  555        cmp	ecx, edx
  556        jl	L_AES_GCM_encrypt_aesni_calc_iv_16_loop
  557        mov	edx, ebx
  558        cmp	ecx, edx
  559        je	L_AES_GCM_encrypt_aesni_calc_iv_done
  560L_AES_GCM_encrypt_aesni_calc_iv_lt16:
  561        sub	rsp, 16
  562        pxor	xmm8, xmm8
  563        xor	ebx, ebx
  564        movdqu	OWORD PTR [rsp], xmm8
  565L_AES_GCM_encrypt_aesni_calc_iv_loop:
  566        movzx	r13d, BYTE PTR [rax+rcx]
  567        mov	BYTE PTR [rsp+rbx], r13b
  568        inc	ecx
  569        inc	ebx
  570        cmp	ecx, edx
  571        jl	L_AES_GCM_encrypt_aesni_calc_iv_loop
  572        movdqu	xmm8, OWORD PTR [rsp]
  573        add	rsp, 16
  574        pshufb	xmm8, OWORD PTR L_aes_gcm_bswap_mask
  575        pxor	xmm4, xmm8
  576        pshufd	xmm1, xmm4, 78
  577        pshufd	xmm2, xmm5, 78
  578        movdqa	xmm3, xmm5
  579        movdqa	xmm0, xmm5
  580        pclmulqdq	xmm3, xmm4, 17
  581        pclmulqdq	xmm0, xmm4, 0
  582        pxor	xmm1, xmm4
  583        pxor	xmm2, xmm5
  584        pclmulqdq	xmm1, xmm2, 0
  585        pxor	xmm1, xmm0
  586        pxor	xmm1, xmm3
  587        movdqa	xmm2, xmm1
  588        movdqa	xmm7, xmm0
  589        movdqa	xmm4, xmm3
  590        pslldq	xmm2, 8
  591        psrldq	xmm1, 8
  592        pxor	xmm7, xmm2
  593        pxor	xmm4, xmm1
  594        movdqa	xmm0, xmm7
  595        movdqa	xmm1, xmm4
  596        psrld	xmm0, 31
  597        psrld	xmm1, 31
  598        pslld	xmm7, 1
  599        pslld	xmm4, 1
  600        movdqa	xmm2, xmm0
  601        pslldq	xmm0, 4
  602        psrldq	xmm2, 12
  603        pslldq	xmm1, 4
  604        por	xmm4, xmm2
  605        por	xmm7, xmm0
  606        por	xmm4, xmm1
  607        movdqa	xmm0, xmm7
  608        movdqa	xmm1, xmm7
  609        movdqa	xmm2, xmm7
  610        pslld	xmm0, 31
  611        pslld	xmm1, 30
  612        pslld	xmm2, 25
  613        pxor	xmm0, xmm1
  614        pxor	xmm0, xmm2
  615        movdqa	xmm1, xmm0
  616        psrldq	xmm1, 4
  617        pslldq	xmm0, 12
  618        pxor	xmm7, xmm0
  619        movdqa	xmm2, xmm7
  620        movdqa	xmm3, xmm7
  621        movdqa	xmm0, xmm7
  622        psrld	xmm2, 1
  623        psrld	xmm3, 2
  624        psrld	xmm0, 7
  625        pxor	xmm2, xmm3
  626        pxor	xmm2, xmm0
  627        pxor	xmm2, xmm1
  628        pxor	xmm2, xmm7
  629        pxor	xmm4, xmm2
  630L_AES_GCM_encrypt_aesni_calc_iv_done:
  631        ; T = Encrypt counter
  632        pxor	xmm0, xmm0
  633        shl	edx, 3
  634        pinsrq	xmm0, rdx, 0
  635        pxor	xmm4, xmm0
  636        pshufd	xmm1, xmm4, 78
  637        pshufd	xmm2, xmm5, 78
  638        movdqa	xmm3, xmm5
  639        movdqa	xmm0, xmm5
  640        pclmulqdq	xmm3, xmm4, 17
  641        pclmulqdq	xmm0, xmm4, 0
  642        pxor	xmm1, xmm4
  643        pxor	xmm2, xmm5
  644        pclmulqdq	xmm1, xmm2, 0
  645        pxor	xmm1, xmm0
  646        pxor	xmm1, xmm3
  647        movdqa	xmm2, xmm1
  648        movdqa	xmm7, xmm0
  649        movdqa	xmm4, xmm3
  650        pslldq	xmm2, 8
  651        psrldq	xmm1, 8
  652        pxor	xmm7, xmm2
  653        pxor	xmm4, xmm1
  654        movdqa	xmm0, xmm7
  655        movdqa	xmm1, xmm4
  656        psrld	xmm0, 31
  657        psrld	xmm1, 31
  658        pslld	xmm7, 1
  659        pslld	xmm4, 1
  660        movdqa	xmm2, xmm0
  661        pslldq	xmm0, 4
  662        psrldq	xmm2, 12
  663        pslldq	xmm1, 4
  664        por	xmm4, xmm2
  665        por	xmm7, xmm0
  666        por	xmm4, xmm1
  667        movdqa	xmm0, xmm7
  668        movdqa	xmm1, xmm7
  669        movdqa	xmm2, xmm7
  670        pslld	xmm0, 31
  671        pslld	xmm1, 30
  672        pslld	xmm2, 25
  673        pxor	xmm0, xmm1
  674        pxor	xmm0, xmm2
  675        movdqa	xmm1, xmm0
  676        psrldq	xmm1, 4
  677        pslldq	xmm0, 12
  678        pxor	xmm7, xmm0
  679        movdqa	xmm2, xmm7
  680        movdqa	xmm3, xmm7
  681        movdqa	xmm0, xmm7
  682        psrld	xmm2, 1
  683        psrld	xmm3, 2
  684        psrld	xmm0, 7
  685        pxor	xmm2, xmm3
  686        pxor	xmm2, xmm0
  687        pxor	xmm2, xmm1
  688        pxor	xmm2, xmm7
  689        pxor	xmm4, xmm2
  690        pshufb	xmm4, OWORD PTR L_aes_gcm_bswap_mask
  691        ;   Encrypt counter
  692        movdqa	xmm8, OWORD PTR [r15]
  693        pxor	xmm8, xmm4
  694        aesenc	xmm8, [r15+16]
  695        aesenc	xmm8, [r15+32]
  696        aesenc	xmm8, [r15+48]
  697        aesenc	xmm8, [r15+64]
  698        aesenc	xmm8, [r15+80]
  699        aesenc	xmm8, [r15+96]
  700        aesenc	xmm8, [r15+112]
  701        aesenc	xmm8, [r15+128]
  702        aesenc	xmm8, [r15+144]
  703        cmp	r10d, 11
  704        movdqa	xmm9, OWORD PTR [r15+160]
  705        jl	L_AES_GCM_encrypt_aesni_calc_iv_2_aesenc_avx_last
  706        aesenc	xmm8, xmm9
  707        aesenc	xmm8, [r15+176]
  708        cmp	r10d, 13
  709        movdqa	xmm9, OWORD PTR [r15+192]
  710        jl	L_AES_GCM_encrypt_aesni_calc_iv_2_aesenc_avx_last
  711        aesenc	xmm8, xmm9
  712        aesenc	xmm8, [r15+208]
  713        movdqa	xmm9, OWORD PTR [r15+224]
  714L_AES_GCM_encrypt_aesni_calc_iv_2_aesenc_avx_last:
  715        aesenclast	xmm8, xmm9
  716        movdqu	OWORD PTR [rsp+144], xmm8
  717L_AES_GCM_encrypt_aesni_iv_done:
  718        ; Additional authentication data
  719        mov	edx, r11d
  720        cmp	edx, 0
  721        je	L_AES_GCM_encrypt_aesni_calc_aad_done
  722        xor	ecx, ecx
  723        cmp	edx, 16
  724        jl	L_AES_GCM_encrypt_aesni_calc_aad_lt16
  725        and	edx, 4294967280
  726L_AES_GCM_encrypt_aesni_calc_aad_16_loop:
  727        movdqu	xmm8, OWORD PTR [r12+rcx]
  728        pshufb	xmm8, OWORD PTR L_aes_gcm_bswap_mask
  729        pxor	xmm6, xmm8
  730        pshufd	xmm1, xmm6, 78
  731        pshufd	xmm2, xmm5, 78
  732        movdqa	xmm3, xmm5
  733        movdqa	xmm0, xmm5
  734        pclmulqdq	xmm3, xmm6, 17
  735        pclmulqdq	xmm0, xmm6, 0
  736        pxor	xmm1, xmm6
  737        pxor	xmm2, xmm5
  738        pclmulqdq	xmm1, xmm2, 0
  739        pxor	xmm1, xmm0
  740        pxor	xmm1, xmm3
  741        movdqa	xmm2, xmm1
  742        movdqa	xmm7, xmm0
  743        movdqa	xmm6, xmm3
  744        pslldq	xmm2, 8
  745        psrldq	xmm1, 8
  746        pxor	xmm7, xmm2
  747        pxor	xmm6, xmm1
  748        movdqa	xmm0, xmm7
  749        movdqa	xmm1, xmm6
  750        psrld	xmm0, 31
  751        psrld	xmm1, 31
  752        pslld	xmm7, 1
  753        pslld	xmm6, 1
  754        movdqa	xmm2, xmm0
  755        pslldq	xmm0, 4
  756        psrldq	xmm2, 12
  757        pslldq	xmm1, 4
  758        por	xmm6, xmm2
  759        por	xmm7, xmm0
  760        por	xmm6, xmm1
  761        movdqa	xmm0, xmm7
  762        movdqa	xmm1, xmm7
  763        movdqa	xmm2, xmm7
  764        pslld	xmm0, 31
  765        pslld	xmm1, 30
  766        pslld	xmm2, 25
  767        pxor	xmm0, xmm1
  768        pxor	xmm0, xmm2
  769        movdqa	xmm1, xmm0
  770        psrldq	xmm1, 4
  771        pslldq	xmm0, 12
  772        pxor	xmm7, xmm0
  773        movdqa	xmm2, xmm7
  774        movdqa	xmm3, xmm7
  775        movdqa	xmm0, xmm7
  776        psrld	xmm2, 1
  777        psrld	xmm3, 2
  778        psrld	xmm0, 7
  779        pxor	xmm2, xmm3
  780        pxor	xmm2, xmm0
  781        pxor	xmm2, xmm1
  782        pxor	xmm2, xmm7
  783        pxor	xmm6, xmm2
  784        add	ecx, 16
  785        cmp	ecx, edx
  786        jl	L_AES_GCM_encrypt_aesni_calc_aad_16_loop
  787        mov	edx, r11d
  788        cmp	ecx, edx
  789        je	L_AES_GCM_encrypt_aesni_calc_aad_done
  790L_AES_GCM_encrypt_aesni_calc_aad_lt16:
  791        sub	rsp, 16
  792        pxor	xmm8, xmm8
  793        xor	ebx, ebx
  794        movdqu	OWORD PTR [rsp], xmm8
  795L_AES_GCM_encrypt_aesni_calc_aad_loop:
  796        movzx	r13d, BYTE PTR [r12+rcx]
  797        mov	BYTE PTR [rsp+rbx], r13b
  798        inc	ecx
  799        inc	ebx
  800        cmp	ecx, edx
  801        jl	L_AES_GCM_encrypt_aesni_calc_aad_loop
  802        movdqu	xmm8, OWORD PTR [rsp]
  803        add	rsp, 16
  804        pshufb	xmm8, OWORD PTR L_aes_gcm_bswap_mask
  805        pxor	xmm6, xmm8
  806        pshufd	xmm1, xmm6, 78
  807        pshufd	xmm2, xmm5, 78
  808        movdqa	xmm3, xmm5
  809        movdqa	xmm0, xmm5
  810        pclmulqdq	xmm3, xmm6, 17
  811        pclmulqdq	xmm0, xmm6, 0
  812        pxor	xmm1, xmm6
  813        pxor	xmm2, xmm5
  814        pclmulqdq	xmm1, xmm2, 0
  815        pxor	xmm1, xmm0
  816        pxor	xmm1, xmm3
  817        movdqa	xmm2, xmm1
  818        movdqa	xmm7, xmm0
  819        movdqa	xmm6, xmm3
  820        pslldq	xmm2, 8
  821        psrldq	xmm1, 8
  822        pxor	xmm7, xmm2
  823        pxor	xmm6, xmm1
  824        movdqa	xmm0, xmm7
  825        movdqa	xmm1, xmm6
  826        psrld	xmm0, 31
  827        psrld	xmm1, 31
  828        pslld	xmm7, 1
  829        pslld	xmm6, 1
  830        movdqa	xmm2, xmm0
  831        pslldq	xmm0, 4
  832        psrldq	xmm2, 12
  833        pslldq	xmm1, 4
  834        por	xmm6, xmm2
  835        por	xmm7, xmm0
  836        por	xmm6, xmm1
  837        movdqa	xmm0, xmm7
  838        movdqa	xmm1, xmm7
  839        movdqa	xmm2, xmm7
  840        pslld	xmm0, 31
  841        pslld	xmm1, 30
  842        pslld	xmm2, 25
  843        pxor	xmm0, xmm1
  844        pxor	xmm0, xmm2
  845        movdqa	xmm1, xmm0
  846        psrldq	xmm1, 4
  847        pslldq	xmm0, 12
  848        pxor	xmm7, xmm0
  849        movdqa	xmm2, xmm7
  850        movdqa	xmm3, xmm7
  851        movdqa	xmm0, xmm7
  852        psrld	xmm2, 1
  853        psrld	xmm3, 2
  854        psrld	xmm0, 7
  855        pxor	xmm2, xmm3
  856        pxor	xmm2, xmm0
  857        pxor	xmm2, xmm1
  858        pxor	xmm2, xmm7
  859        pxor	xmm6, xmm2
  860L_AES_GCM_encrypt_aesni_calc_aad_done:
  861        ; Calculate counter and H
  862        pshufb	xmm4, OWORD PTR L_aes_gcm_bswap_epi64
  863        movdqa	xmm9, xmm5
  864        paddd	xmm4, OWORD PTR L_aes_gcm_one
  865        movdqa	xmm8, xmm5
  866        movdqu	OWORD PTR [rsp+128], xmm4
  867        psrlq	xmm9, 63
  868        psllq	xmm8, 1
  869        pslldq	xmm9, 8
  870        por	xmm8, xmm9
  871        pshufd	xmm5, xmm5, 255
  872        psrad	xmm5, 31
  873        pand	xmm5, OWORD PTR L_aes_gcm_mod2_128
  874        pxor	xmm5, xmm8
  875        xor	rbx, rbx
  876        cmp	r9d, 128
  877        mov	r13d, r9d
  878        jl	L_AES_GCM_encrypt_aesni_done_128
  879        and	r13d, 4294967168
  880        movdqa	xmm2, xmm6
  881        ; H ^ 1
  882        movdqu	OWORD PTR [rsp], xmm5
  883        ; H ^ 2
  884        pshufd	xmm9, xmm5, 78
  885        pshufd	xmm10, xmm5, 78
  886        movdqa	xmm11, xmm5
  887        movdqa	xmm8, xmm5
  888        pclmulqdq	xmm11, xmm5, 17
  889        pclmulqdq	xmm8, xmm5, 0
  890        pxor	xmm9, xmm5
  891        pxor	xmm10, xmm5
  892        pclmulqdq	xmm9, xmm10, 0
  893        pxor	xmm9, xmm8
  894        pxor	xmm9, xmm11
  895        movdqa	xmm10, xmm9
  896        movdqa	xmm0, xmm11
  897        pslldq	xmm10, 8
  898        psrldq	xmm9, 8
  899        pxor	xmm8, xmm10
  900        pxor	xmm0, xmm9
  901        movdqa	xmm12, xmm8
  902        movdqa	xmm13, xmm8
  903        movdqa	xmm14, xmm8
  904        pslld	xmm12, 31
  905        pslld	xmm13, 30
  906        pslld	xmm14, 25
  907        pxor	xmm12, xmm13
  908        pxor	xmm12, xmm14
  909        movdqa	xmm13, xmm12
  910        psrldq	xmm13, 4
  911        pslldq	xmm12, 12
  912        pxor	xmm8, xmm12
  913        movdqa	xmm14, xmm8
  914        movdqa	xmm10, xmm8
  915        movdqa	xmm9, xmm8
  916        psrld	xmm14, 1
  917        psrld	xmm10, 2
  918        psrld	xmm9, 7
  919        pxor	xmm14, xmm10
  920        pxor	xmm14, xmm9
  921        pxor	xmm14, xmm13
  922        pxor	xmm14, xmm8
  923        pxor	xmm0, xmm14
  924        movdqu	OWORD PTR [rsp+16], xmm0
  925        ; H ^ 3
  926        pshufd	xmm9, xmm5, 78
  927        pshufd	xmm10, xmm0, 78
  928        movdqa	xmm11, xmm0
  929        movdqa	xmm8, xmm0
  930        pclmulqdq	xmm11, xmm5, 17
  931        pclmulqdq	xmm8, xmm5, 0
  932        pxor	xmm9, xmm5
  933        pxor	xmm10, xmm0
  934        pclmulqdq	xmm9, xmm10, 0
  935        pxor	xmm9, xmm8
  936        pxor	xmm9, xmm11
  937        movdqa	xmm10, xmm9
  938        movdqa	xmm1, xmm11
  939        pslldq	xmm10, 8
  940        psrldq	xmm9, 8
  941        pxor	xmm8, xmm10
  942        pxor	xmm1, xmm9
  943        movdqa	xmm12, xmm8
  944        movdqa	xmm13, xmm8
  945        movdqa	xmm14, xmm8
  946        pslld	xmm12, 31
  947        pslld	xmm13, 30
  948        pslld	xmm14, 25
  949        pxor	xmm12, xmm13
  950        pxor	xmm12, xmm14
  951        movdqa	xmm13, xmm12
  952        psrldq	xmm13, 4
  953        pslldq	xmm12, 12
  954        pxor	xmm8, xmm12
  955        movdqa	xmm14, xmm8
  956        movdqa	xmm10, xmm8
  957        movdqa	xmm9, xmm8
  958        psrld	xmm14, 1
  959        psrld	xmm10, 2
  960        psrld	xmm9, 7
  961        pxor	xmm14, xmm10
  962        pxor	xmm14, xmm9
  963        pxor	xmm14, xmm13
  964        pxor	xmm14, xmm8
  965        pxor	xmm1, xmm14
  966        movdqu	OWORD PTR [rsp+32], xmm1
  967        ; H ^ 4
  968        pshufd	xmm9, xmm0, 78
  969        pshufd	xmm10, xmm0, 78
  970        movdqa	xmm11, xmm0
  971        movdqa	xmm8, xmm0
  972        pclmulqdq	xmm11, xmm0, 17
  973        pclmulqdq	xmm8, xmm0, 0
  974        pxor	xmm9, xmm0
  975        pxor	xmm10, xmm0
  976        pclmulqdq	xmm9, xmm10, 0
  977        pxor	xmm9, xmm8
  978        pxor	xmm9, xmm11
  979        movdqa	xmm10, xmm9
  980        movdqa	xmm3, xmm11
  981        pslldq	xmm10, 8
  982        psrldq	xmm9, 8
  983        pxor	xmm8, xmm10
  984        pxor	xmm3, xmm9
  985        movdqa	xmm12, xmm8
  986        movdqa	xmm13, xmm8
  987        movdqa	xmm14, xmm8
  988        pslld	xmm12, 31
  989        pslld	xmm13, 30
  990        pslld	xmm14, 25
  991        pxor	xmm12, xmm13
  992        pxor	xmm12, xmm14
  993        movdqa	xmm13, xmm12
  994        psrldq	xmm13, 4
  995        pslldq	xmm12, 12
  996        pxor	xmm8, xmm12
  997        movdqa	xmm14, xmm8
  998        movdqa	xmm10, xmm8
  999        movdqa	xmm9, xmm8
 1000        psrld	xmm14, 1
 1001        psrld	xmm10, 2
 1002        psrld	xmm9, 7
 1003        pxor	xmm14, xmm10
 1004        pxor	xmm14, xmm9
 1005        pxor	xmm14, xmm13
 1006        pxor	xmm14, xmm8
 1007        pxor	xmm3, xmm14
 1008        movdqu	OWORD PTR [rsp+48], xmm3
 1009        ; H ^ 5
 1010        pshufd	xmm9, xmm0, 78
 1011        pshufd	xmm10, xmm1, 78
 1012        movdqa	xmm11, xmm1
 1013        movdqa	xmm8, xmm1
 1014        pclmulqdq	xmm11, xmm0, 17
 1015        pclmulqdq	xmm8, xmm0, 0
 1016        pxor	xmm9, xmm0
 1017        pxor	xmm10, xmm1
 1018        pclmulqdq	xmm9, xmm10, 0
 1019        pxor	xmm9, xmm8
 1020        pxor	xmm9, xmm11
 1021        movdqa	xmm10, xmm9
 1022        movdqa	xmm7, xmm11
 1023        pslldq	xmm10, 8
 1024        psrldq	xmm9, 8
 1025        pxor	xmm8, xmm10
 1026        pxor	xmm7, xmm9
 1027        movdqa	xmm12, xmm8
 1028        movdqa	xmm13, xmm8
 1029        movdqa	xmm14, xmm8
 1030        pslld	xmm12, 31
 1031        pslld	xmm13, 30
 1032        pslld	xmm14, 25
 1033        pxor	xmm12, xmm13
 1034        pxor	xmm12, xmm14
 1035        movdqa	xmm13, xmm12
 1036        psrldq	xmm13, 4
 1037        pslldq	xmm12, 12
 1038        pxor	xmm8, xmm12
 1039        movdqa	xmm14, xmm8
 1040        movdqa	xmm10, xmm8
 1041        movdqa	xmm9, xmm8
 1042        psrld	xmm14, 1
 1043        psrld	xmm10, 2
 1044        psrld	xmm9, 7
 1045        pxor	xmm14, xmm10
 1046        pxor	xmm14, xmm9
 1047        pxor	xmm14, xmm13
 1048        pxor	xmm14, xmm8
 1049        pxor	xmm7, xmm14
 1050        movdqu	OWORD PTR [rsp+64], xmm7
 1051        ; H ^ 6
 1052        pshufd	xmm9, xmm1, 78
 1053        pshufd	xmm10, xmm1, 78
 1054        movdqa	xmm11, xmm1
 1055        movdqa	xmm8, xmm1
 1056        pclmulqdq	xmm11, xmm1, 17
 1057        pclmulqdq	xmm8, xmm1, 0
 1058        pxor	xmm9, xmm1
 1059        pxor	xmm10, xmm1
 1060        pclmulqdq	xmm9, xmm10, 0
 1061        pxor	xmm9, xmm8
 1062        pxor	xmm9, xmm11
 1063        movdqa	xmm10, xmm9
 1064        movdqa	xmm7, xmm11
 1065        pslldq	xmm10, 8
 1066        psrldq	xmm9, 8
 1067        pxor	xmm8, xmm10
 1068        pxor	xmm7, xmm9
 1069        movdqa	xmm12, xmm8
 1070        movdqa	xmm13, xmm8
 1071        movdqa	xmm14, xmm8
 1072        pslld	xmm12, 31
 1073        pslld	xmm13, 30
 1074        pslld	xmm14, 25
 1075        pxor	xmm12, xmm13
 1076        pxor	xmm12, xmm14
 1077        movdqa	xmm13, xmm12
 1078        psrldq	xmm13, 4
 1079        pslldq	xmm12, 12
 1080        pxor	xmm8, xmm12
 1081        movdqa	xmm14, xmm8
 1082        movdqa	xmm10, xmm8
 1083        movdqa	xmm9, xmm8
 1084        psrld	xmm14, 1
 1085        psrld	xmm10, 2
 1086        psrld	xmm9, 7
 1087        pxor	xmm14, xmm10
 1088        pxor	xmm14, xmm9
 1089        pxor	xmm14, xmm13
 1090        pxor	xmm14, xmm8
 1091        pxor	xmm7, xmm14
 1092        movdqu	OWORD PTR [rsp+80], xmm7
 1093        ; H ^ 7
 1094        pshufd	xmm9, xmm1, 78
 1095        pshufd	xmm10, xmm3, 78
 1096        movdqa	xmm11, xmm3
 1097        movdqa	xmm8, xmm3
 1098        pclmulqdq	xmm11, xmm1, 17
 1099        pclmulqdq	xmm8, xmm1, 0
 1100        pxor	xmm9, xmm1
 1101        pxor	xmm10, xmm3
 1102        pclmulqdq	xmm9, xmm10, 0
 1103        pxor	xmm9, xmm8
 1104        pxor	xmm9, xmm11
 1105        movdqa	xmm10, xmm9
 1106        movdqa	xmm7, xmm11
 1107        pslldq	xmm10, 8
 1108        psrldq	xmm9, 8
 1109        pxor	xmm8, xmm10
 1110        pxor	xmm7, xmm9
 1111        movdqa	xmm12, xmm8
 1112        movdqa	xmm13, xmm8
 1113        movdqa	xmm14, xmm8
 1114        pslld	xmm12, 31
 1115        pslld	xmm13, 30
 1116        pslld	xmm14, 25
 1117        pxor	xmm12, xmm13
 1118        pxor	xmm12, xmm14
 1119        movdqa	xmm13, xmm12
 1120        psrldq	xmm13, 4
 1121        pslldq	xmm12, 12
 1122        pxor	xmm8, xmm12
 1123        movdqa	xmm14, xmm8
 1124        movdqa	xmm10, xmm8
 1125        movdqa	xmm9, xmm8
 1126        psrld	xmm14, 1
 1127        psrld	xmm10, 2
 1128        psrld	xmm9, 7
 1129        pxor	xmm14, xmm10
 1130        pxor	xmm14, xmm9
 1131        pxor	xmm14, xmm13
 1132        pxor	xmm14, xmm8
 1133        pxor	xmm7, xmm14
 1134        movdqu	OWORD PTR [rsp+96], xmm7
 1135        ; H ^ 8
 1136        pshufd	xmm9, xmm3, 78
 1137        pshufd	xmm10, xmm3, 78
 1138        movdqa	xmm11, xmm3
 1139        movdqa	xmm8, xmm3
 1140        pclmulqdq	xmm11, xmm3, 17
 1141        pclmulqdq	xmm8, xmm3, 0
 1142        pxor	xmm9, xmm3
 1143        pxor	xmm10, xmm3
 1144        pclmulqdq	xmm9, xmm10, 0
 1145        pxor	xmm9, xmm8
 1146        pxor	xmm9, xmm11
 1147        movdqa	xmm10, xmm9
 1148        movdqa	xmm7, xmm11
 1149        pslldq	xmm10, 8
 1150        psrldq	xmm9, 8
 1151        pxor	xmm8, xmm10
 1152        pxor	xmm7, xmm9
 1153        movdqa	xmm12, xmm8
 1154        movdqa	xmm13, xmm8
 1155        movdqa	xmm14, xmm8
 1156        pslld	xmm12, 31
 1157        pslld	xmm13, 30
 1158        pslld	xmm14, 25
 1159        pxor	xmm12, xmm13
 1160        pxor	xmm12, xmm14
 1161        movdqa	xmm13, xmm12
 1162        psrldq	xmm13, 4
 1163        pslldq	xmm12, 12
 1164        pxor	xmm8, xmm12
 1165        movdqa	xmm14, xmm8
 1166        movdqa	xmm10, xmm8
 1167        movdqa	xmm9, xmm8
 1168        psrld	xmm14, 1
 1169        psrld	xmm10, 2
 1170        psrld	xmm9, 7
 1171        pxor	xmm14, xmm10
 1172        pxor	xmm14, xmm9
 1173        pxor	xmm14, xmm13
 1174        pxor	xmm14, xmm8
 1175        pxor	xmm7, xmm14
 1176        movdqu	OWORD PTR [rsp+112], xmm7
 1177        ; First 128 bytes of input
 1178        movdqu	xmm8, OWORD PTR [rsp+128]
 1179        movdqa	xmm1, OWORD PTR L_aes_gcm_bswap_epi64
 1180        movdqa	xmm0, xmm8
 1181        pshufb	xmm8, xmm1
 1182        movdqa	xmm9, xmm0
 1183        paddd	xmm9, OWORD PTR L_aes_gcm_one
 1184        pshufb	xmm9, xmm1
 1185        movdqa	xmm10, xmm0
 1186        paddd	xmm10, OWORD PTR L_aes_gcm_two
 1187        pshufb	xmm10, xmm1
 1188        movdqa	xmm11, xmm0
 1189        paddd	xmm11, OWORD PTR L_aes_gcm_three
 1190        pshufb	xmm11, xmm1
 1191        movdqa	xmm12, xmm0
 1192        paddd	xmm12, OWORD PTR L_aes_gcm_four
 1193        pshufb	xmm12, xmm1
 1194        movdqa	xmm13, xmm0
 1195        paddd	xmm13, OWORD PTR L_aes_gcm_five
 1196        pshufb	xmm13, xmm1
 1197        movdqa	xmm14, xmm0
 1198        paddd	xmm14, OWORD PTR L_aes_gcm_six
 1199        pshufb	xmm14, xmm1
 1200        movdqa	xmm15, xmm0
 1201        paddd	xmm15, OWORD PTR L_aes_gcm_seven
 1202        pshufb	xmm15, xmm1
 1203        paddd	xmm0, OWORD PTR L_aes_gcm_eight
 1204        movdqa	xmm7, OWORD PTR [r15]
 1205        movdqu	OWORD PTR [rsp+128], xmm0
 1206        pxor	xmm8, xmm7
 1207        pxor	xmm9, xmm7
 1208        pxor	xmm10, xmm7
 1209        pxor	xmm11, xmm7
 1210        pxor	xmm12, xmm7
 1211        pxor	xmm13, xmm7
 1212        pxor	xmm14, xmm7
 1213        pxor	xmm15, xmm7
 1214        movdqa	xmm7, OWORD PTR [r15+16]
 1215        aesenc	xmm8, xmm7
 1216        aesenc	xmm9, xmm7
 1217        aesenc	xmm10, xmm7
 1218        aesenc	xmm11, xmm7
 1219        aesenc	xmm12, xmm7
 1220        aesenc	xmm13, xmm7
 1221        aesenc	xmm14, xmm7
 1222        aesenc	xmm15, xmm7
 1223        movdqa	xmm7, OWORD PTR [r15+32]
 1224        aesenc	xmm8, xmm7
 1225        aesenc	xmm9, xmm7
 1226        aesenc	xmm10, xmm7
 1227        aesenc	xmm11, xmm7
 1228        aesenc	xmm12, xmm7
 1229        aesenc	xmm13, xmm7
 1230        aesenc	xmm14, xmm7
 1231        aesenc	xmm15, xmm7
 1232        movdqa	xmm7, OWORD PTR [r15+48]
 1233        aesenc	xmm8, xmm7
 1234        aesenc	xmm9, xmm7
 1235        aesenc	xmm10, xmm7
 1236        aesenc	xmm11, xmm7
 1237        aesenc	xmm12, xmm7
 1238        aesenc	xmm13, xmm7
 1239        aesenc	xmm14, xmm7
 1240        aesenc	xmm15, xmm7
 1241        movdqa	xmm7, OWORD PTR [r15+64]
 1242        aesenc	xmm8, xmm7
 1243        aesenc	xmm9, xmm7
 1244        aesenc	xmm10, xmm7
 1245        aesenc	xmm11, xmm7
 1246        aesenc	xmm12, xmm7
 1247        aesenc	xmm13, xmm7
 1248        aesenc	xmm14, xmm7
 1249        aesenc	xmm15, xmm7
 1250        movdqa	xmm7, OWORD PTR [r15+80]
 1251        aesenc	xmm8, xmm7
 1252        aesenc	xmm9, xmm7
 1253        aesenc	xmm10, xmm7
 1254        aesenc	xmm11, xmm7
 1255        aesenc	xmm12, xmm7
 1256        aesenc	xmm13, xmm7
 1257        aesenc	xmm14, xmm7
 1258        aesenc	xmm15, xmm7
 1259        movdqa	xmm7, OWORD PTR [r15+96]
 1260        aesenc	xmm8, xmm7
 1261        aesenc	xmm9, xmm7
 1262        aesenc	xmm10, xmm7
 1263        aesenc	xmm11, xmm7
 1264        aesenc	xmm12, xmm7
 1265        aesenc	xmm13, xmm7
 1266        aesenc	xmm14, xmm7
 1267        aesenc	xmm15, xmm7
 1268        movdqa	xmm7, OWORD PTR [r15+112]
 1269        aesenc	xmm8, xmm7
 1270        aesenc	xmm9, xmm7
 1271        aesenc	xmm10, xmm7
 1272        aesenc	xmm11, xmm7
 1273        aesenc	xmm12, xmm7
 1274        aesenc	xmm13, xmm7
 1275        aesenc	xmm14, xmm7
 1276        aesenc	xmm15, xmm7
 1277        movdqa	xmm7, OWORD PTR [r15+128]
 1278        aesenc	xmm8, xmm7
 1279        aesenc	xmm9, xmm7
 1280        aesenc	xmm10, xmm7
 1281        aesenc	xmm11, xmm7
 1282        aesenc	xmm12, xmm7
 1283        aesenc	xmm13, xmm7
 1284        aesenc	xmm14, xmm7
 1285        aesenc	xmm15, xmm7
 1286        movdqa	xmm7, OWORD PTR [r15+144]
 1287        aesenc	xmm8, xmm7
 1288        aesenc	xmm9, xmm7
 1289        aesenc	xmm10, xmm7
 1290        aesenc	xmm11, xmm7
 1291        aesenc	xmm12, xmm7
 1292        aesenc	xmm13, xmm7
 1293        aesenc	xmm14, xmm7
 1294        aesenc	xmm15, xmm7
 1295        cmp	r10d, 11
 1296        movdqa	xmm7, OWORD PTR [r15+160]
 1297        jl	L_AES_GCM_encrypt_aesni_enc_done
 1298        aesenc	xmm8, xmm7
 1299        aesenc	xmm9, xmm7
 1300        aesenc	xmm10, xmm7
 1301        aesenc	xmm11, xmm7
 1302        aesenc	xmm12, xmm7
 1303        aesenc	xmm13, xmm7
 1304        aesenc	xmm14, xmm7
 1305        aesenc	xmm15, xmm7
 1306        movdqa	xmm7, OWORD PTR [r15+176]
 1307        aesenc	xmm8, xmm7
 1308        aesenc	xmm9, xmm7
 1309        aesenc	xmm10, xmm7
 1310        aesenc	xmm11, xmm7
 1311        aesenc	xmm12, xmm7
 1312        aesenc	xmm13, xmm7
 1313        aesenc	xmm14, xmm7
 1314        aesenc	xmm15, xmm7
 1315        cmp	r10d, 13
 1316        movdqa	xmm7, OWORD PTR [r15+192]
 1317        jl	L_AES_GCM_encrypt_aesni_enc_done
 1318        aesenc	xmm8, xmm7
 1319        aesenc	xmm9, xmm7
 1320        aesenc	xmm10, xmm7
 1321        aesenc	xmm11, xmm7
 1322        aesenc	xmm12, xmm7
 1323        aesenc	xmm13, xmm7
 1324        aesenc	xmm14, xmm7
 1325        aesenc	xmm15, xmm7
 1326        movdqa	xmm7, OWORD PTR [r15+208]
 1327        aesenc	xmm8, xmm7
 1328        aesenc	xmm9, xmm7
 1329        aesenc	xmm10, xmm7
 1330        aesenc	xmm11, xmm7
 1331        aesenc	xmm12, xmm7
 1332        aesenc	xmm13, xmm7
 1333        aesenc	xmm14, xmm7
 1334        aesenc	xmm15, xmm7
 1335        movdqa	xmm7, OWORD PTR [r15+224]
 1336L_AES_GCM_encrypt_aesni_enc_done:
 1337        aesenclast	xmm8, xmm7
 1338        aesenclast	xmm9, xmm7
 1339        movdqu	xmm0, OWORD PTR [rdi]
 1340        movdqu	xmm1, OWORD PTR [rdi+16]
 1341        pxor	xmm8, xmm0
 1342        pxor	xmm9, xmm1
 1343        movdqu	OWORD PTR [rsi], xmm8
 1344        movdqu	OWORD PTR [rsi+16], xmm9
 1345        aesenclast	xmm10, xmm7
 1346        aesenclast	xmm11, xmm7
 1347        movdqu	xmm0, OWORD PTR [rdi+32]
 1348        movdqu	xmm1, OWORD PTR [rdi+48]
 1349        pxor	xmm10, xmm0
 1350        pxor	xmm11, xmm1
 1351        movdqu	OWORD PTR [rsi+32], xmm10
 1352        movdqu	OWORD PTR [rsi+48], xmm11
 1353        aesenclast	xmm12, xmm7
 1354        aesenclast	xmm13, xmm7
 1355        movdqu	xmm0, OWORD PTR [rdi+64]
 1356        movdqu	xmm1, OWORD PTR [rdi+80]
 1357        pxor	xmm12, xmm0
 1358        pxor	xmm13, xmm1
 1359        movdqu	OWORD PTR [rsi+64], xmm12
 1360        movdqu	OWORD PTR [rsi+80], xmm13
 1361        aesenclast	xmm14, xmm7
 1362        aesenclast	xmm15, xmm7
 1363        movdqu	xmm0, OWORD PTR [rdi+96]
 1364        movdqu	xmm1, OWORD PTR [rdi+112]
 1365        pxor	xmm14, xmm0
 1366        pxor	xmm15, xmm1
 1367        movdqu	OWORD PTR [rsi+96], xmm14
 1368        movdqu	OWORD PTR [rsi+112], xmm15
 1369        cmp	r13d, 128
 1370        mov	ebx, 128
 1371        jle	L_AES_GCM_encrypt_aesni_end_128
 1372        ; More 128 bytes of input
 1373L_AES_GCM_encrypt_aesni_ghash_128:
 1374        lea	rcx, QWORD PTR [rdi+rbx]
 1375        lea	rdx, QWORD PTR [rsi+rbx]
 1376        movdqu	xmm8, OWORD PTR [rsp+128]
 1377        movdqa	xmm1, OWORD PTR L_aes_gcm_bswap_epi64
 1378        movdqa	xmm0, xmm8
 1379        pshufb	xmm8, xmm1
 1380        movdqa	xmm9, xmm0
 1381        paddd	xmm9, OWORD PTR L_aes_gcm_one
 1382        pshufb	xmm9, xmm1
 1383        movdqa	xmm10, xmm0
 1384        paddd	xmm10, OWORD PTR L_aes_gcm_two
 1385        pshufb	xmm10, xmm1
 1386        movdqa	xmm11, xmm0
 1387        paddd	xmm11, OWORD PTR L_aes_gcm_three
 1388        pshufb	xmm11, xmm1
 1389        movdqa	xmm12, xmm0
 1390        paddd	xmm12, OWORD PTR L_aes_gcm_four
 1391        pshufb	xmm12, xmm1
 1392        movdqa	xmm13, xmm0
 1393        paddd	xmm13, OWORD PTR L_aes_gcm_five
 1394        pshufb	xmm13, xmm1
 1395        movdqa	xmm14, xmm0
 1396        paddd	xmm14, OWORD PTR L_aes_gcm_six
 1397        pshufb	xmm14, xmm1
 1398        movdqa	xmm15, xmm0
 1399        paddd	xmm15, OWORD PTR L_aes_gcm_seven
 1400        pshufb	xmm15, xmm1
 1401        paddd	xmm0, OWORD PTR L_aes_gcm_eight
 1402        movdqa	xmm7, OWORD PTR [r15]
 1403        movdqu	OWORD PTR [rsp+128], xmm0
 1404        pxor	xmm8, xmm7
 1405        pxor	xmm9, xmm7
 1406        pxor	xmm10, xmm7
 1407        pxor	xmm11, xmm7
 1408        pxor	xmm12, xmm7
 1409        pxor	xmm13, xmm7
 1410        pxor	xmm14, xmm7
 1411        pxor	xmm15, xmm7
 1412        movdqu	xmm7, OWORD PTR [rsp+112]
 1413        movdqu	xmm0, OWORD PTR [rdx+-128]
 1414        aesenc	xmm8, [r15+16]
 1415        pshufb	xmm0, OWORD PTR L_aes_gcm_bswap_mask
 1416        pxor	xmm0, xmm2
 1417        pshufd	xmm1, xmm7, 78
 1418        pshufd	xmm5, xmm0, 78
 1419        pxor	xmm1, xmm7
 1420        pxor	xmm5, xmm0
 1421        movdqa	xmm3, xmm0
 1422        pclmulqdq	xmm3, xmm7, 17
 1423        aesenc	xmm9, [r15+16]
 1424        aesenc	xmm10, [r15+16]
 1425        movdqa	xmm2, xmm0
 1426        pclmulqdq	xmm2, xmm7, 0
 1427        aesenc	xmm11, [r15+16]
 1428        aesenc	xmm12, [r15+16]
 1429        pclmulqdq	xmm1, xmm5, 0
 1430        aesenc	xmm13, [r15+16]
 1431        aesenc	xmm14, [r15+16]
 1432        aesenc	xmm15, [r15+16]
 1433        pxor	xmm1, xmm2
 1434        pxor	xmm1, xmm3
 1435        movdqu	xmm7, OWORD PTR [rsp+96]
 1436        movdqu	xmm0, OWORD PTR [rdx+-112]
 1437        pshufd	xmm4, xmm7, 78
 1438        pshufb	xmm0, OWORD PTR L_aes_gcm_bswap_mask
 1439        aesenc	xmm8, [r15+32]
 1440        pxor	xmm4, xmm7
 1441        pshufd	xmm5, xmm0, 78
 1442        pxor	xmm5, xmm0
 1443        movdqa	xmm6, xmm0
 1444        pclmulqdq	xmm6, xmm7, 17
 1445        aesenc	xmm9, [r15+32]
 1446        aesenc	xmm10, [r15+32]
 1447        pclmulqdq	xmm7, xmm0, 0
 1448        aesenc	xmm11, [r15+32]
 1449        aesenc	xmm12, [r15+32]
 1450        pclmulqdq	xmm4, xmm5, 0
 1451        aesenc	xmm13, [r15+32]
 1452        aesenc	xmm14, [r15+32]
 1453        aesenc	xmm15, [r15+32]
 1454        pxor	xmm1, xmm7
 1455        pxor	xmm2, xmm7
 1456        pxor	xmm1, xmm6
 1457        pxor	xmm3, xmm6
 1458        pxor	xmm1, xmm4
 1459        movdqu	xmm7, OWORD PTR [rsp+80]
 1460        movdqu	xmm0, OWORD PTR [rdx+-96]
 1461        pshufd	xmm4, xmm7, 78
 1462        pshufb	xmm0, OWORD PTR L_aes_gcm_bswap_mask
 1463        aesenc	xmm8, [r15+48]
 1464        pxor	xmm4, xmm7
 1465        pshufd	xmm5, xmm0, 78
 1466        pxor	xmm5, xmm0
 1467        movdqa	xmm6, xmm0
 1468        pclmulqdq	xmm6, xmm7, 17
 1469        aesenc	xmm9, [r15+48]
 1470        aesenc	xmm10, [r15+48]
 1471        pclmulqdq	xmm7, xmm0, 0
 1472        aesenc	xmm11, [r15+48]
 1473        aesenc	xmm12, [r15+48]
 1474        pclmulqdq	xmm4, xmm5, 0
 1475        aesenc	xmm13, [r15+48]
 1476        aesenc	xmm14, [r15+48]
 1477        aesenc	xmm15, [r15+48]
 1478        pxor	xmm1, xmm7
 1479        pxor	xmm2, xmm7
 1480        pxor	xmm1, xmm6
 1481        pxor	xmm3, xmm6
 1482        pxor	xmm1, xmm4
 1483        movdqu	xmm7, OWORD PTR [rsp+64]
 1484        movdqu	xmm0, OWORD PTR [rdx+-80]
 1485        pshufd	xmm4, xmm7, 78
 1486        pshufb	xmm0, OWORD PTR L_aes_gcm_bswap_mask
 1487        aesenc	xmm8, [r15+64]
 1488        pxor	xmm4, xmm7
 1489        pshufd	xmm5, xmm0, 78
 1490        pxor	xmm5, xmm0
 1491        movdqa	xmm6, xmm0
 1492        pclmulqdq	xmm6, xmm7, 17
 1493        aesenc	xmm9, [r15+64]
 1494        aesenc	xmm10, [r15+64]
 1495        pclmulqdq	xmm7, xmm0, 0
 1496        aesenc	xmm11, [r15+64]
 1497        aesenc	xmm12, [r15+64]
 1498        pclmulqdq	xmm4, xmm5, 0
 1499        aesenc	xmm13, [r15+64]
 1500        aesenc	xmm14, [r15+64]
 1501        aesenc	xmm15, [r15+64]
 1502        pxor	xmm1, xmm7
 1503        pxor	xmm2, xmm7
 1504        pxor	xmm1, xmm6
 1505        pxor	xmm3, xmm6
 1506        pxor	xmm1, xmm4
 1507        movdqu	xmm7, OWORD PTR [rsp+48]
 1508        movdqu	xmm0, OWORD PTR [rdx+-64]
 1509        pshufd	xmm4, xmm7, 78
 1510        pshufb	xmm0, OWORD PTR L_aes_gcm_bswap_mask
 1511        aesenc	xmm8, [r15+80]
 1512        pxor	xmm4, xmm7
 1513        pshufd	xmm5, xmm0, 78
 1514        pxor	xmm5, xmm0
 1515        movdqa	xmm6, xmm0
 1516        pclmulqdq	xmm6, xmm7, 17
 1517        aesenc	xmm9, [r15+80]
 1518        aesenc	xmm10, [r15+80]
 1519        pclmulqdq	xmm7, xmm0, 0
 1520        aesenc	xmm11, [r15+80]
 1521        aesenc	xmm12, [r15+80]
 1522        pclmulqdq	xmm4, xmm5, 0
 1523        aesenc	xmm13, [r15+80]
 1524        aesenc	xmm14, [r15+80]
 1525        aesenc	xmm15, [r15+80]
 1526        pxor	xmm1, xmm7
 1527        pxor	xmm2, xmm7
 1528        pxor	xmm1, xmm6
 1529        pxor	xmm3, xmm6
 1530        pxor	xmm1, xmm4
 1531        movdqu	xmm7, OWORD PTR [rsp+32]
 1532        movdqu	xmm0, OWORD PTR [rdx+-48]
 1533        pshufd	xmm4, xmm7, 78
 1534        pshufb	xmm0, OWORD PTR L_aes_gcm_bswap_mask
 1535        aesenc	xmm8, [r15+96]
 1536        pxor	xmm4, xmm7
 1537        pshufd	xmm5, xmm0, 78
 1538        pxor	xmm5, xmm0
 1539        movdqa	xmm6, xmm0
 1540        pclmulqdq	xmm6, xmm7, 17
 1541        aesenc	xmm9, [r15+96]
 1542        aesenc	xmm10, [r15+96]
 1543        pclmulqdq	xmm7, xmm0, 0
 1544        aesenc	xmm11, [r15+96]
 1545        aesenc	xmm12, [r15+96]
 1546        pclmulqdq	xmm4, xmm5, 0
 1547        aesenc	xmm13, [r15+96]
 1548        aesenc	xmm14, [r15+96]
 1549        aesenc	xmm15, [r15+96]
 1550        pxor	xmm1, xmm7
 1551        pxor	xmm2, xmm7
 1552        pxor	xmm1, xmm6
 1553        pxor	xmm3, xmm6
 1554        pxor	xmm1, xmm4
 1555        movdqu	xmm7, OWORD PTR [rsp+16]
 1556        movdqu	xmm0, OWORD PTR [rdx+-32]
 1557        pshufd	xmm4, xmm7, 78
 1558        pshufb	xmm0, OWORD PTR L_aes_gcm_bswap_mask
 1559        aesenc	xmm8, [r15+112]
 1560        pxor	xmm4, xmm7
 1561        pshufd	xmm5, xmm0, 78
 1562        pxor	xmm5, xmm0
 1563        movdqa	xmm6, xmm0
 1564        pclmulqdq	xmm6, xmm7, 17
 1565        aesenc	xmm9, [r15+112]
 1566        aesenc	xmm10, [r15+112]
 1567        pclmulqdq	xmm7, xmm0, 0
 1568        aesenc	xmm11, [r15+112]
 1569        aesenc	xmm12, [r15+112]
 1570        pclmulqdq	xmm4, xmm5, 0
 1571        aesenc	xmm13, [r15+112]
 1572        aesenc	xmm14, [r15+112]
 1573        aesenc	xmm15, [r15+112]
 1574        pxor	xmm1, xmm7
 1575        pxor	xmm2, xmm7
 1576        pxor	xmm1, xmm6
 1577        pxor	xmm3, xmm6
 1578        pxor	xmm1, xmm4
 1579        movdqu	xmm7, OWORD PTR [rsp]
 1580        movdqu	xmm0, OWORD PTR [rdx+-16]
 1581        pshufd	xmm4, xmm7, 78
 1582        pshufb	xmm0, OWORD PTR L_aes_gcm_bswap_mask
 1583        aesenc	xmm8, [r15+128]
 1584        pxor	xmm4, xmm7
 1585        pshufd	xmm5, xmm0, 78
 1586        pxor	xmm5, xmm0
 1587        movdqa	xmm6, xmm0
 1588        pclmulqdq	xmm6, xmm7, 17
 1589        aesenc	xmm9, [r15+128]
 1590        aesenc	xmm10, [r15+128]
 1591        pclmulqdq	xmm7, xmm0, 0
 1592        aesenc	xmm11, [r15+128]
 1593        aesenc	xmm12, [r15+128]
 1594        pclmulqdq	xmm4, xmm5, 0
 1595        aesenc	xmm13, [r15+128]
 1596        aesenc	xmm14, [r15+128]
 1597        aesenc	xmm15, [r15+128]
 1598        pxor	xmm1, xmm7
 1599        pxor	xmm2, xmm7
 1600        pxor	xmm1, xmm6
 1601        pxor	xmm3, xmm6
 1602        pxor	xmm1, xmm4
 1603        movdqa	xmm5, xmm1
 1604        psrldq	xmm1, 8
 1605        pslldq	xmm5, 8
 1606        aesenc	xmm8, [r15+144]
 1607        pxor	xmm2, xmm5
 1608        pxor	xmm3, xmm1
 1609        movdqa	xmm7, xmm2
 1610        movdqa	xmm4, xmm2
 1611        movdqa	xmm5, xmm2
 1612        aesenc	xmm9, [r15+144]
 1613        pslld	xmm7, 31
 1614        pslld	xmm4, 30
 1615        pslld	xmm5, 25
 1616        aesenc	xmm10, [r15+144]
 1617        pxor	xmm7, xmm4
 1618        pxor	xmm7, xmm5
 1619        aesenc	xmm11, [r15+144]
 1620        movdqa	xmm4, xmm7
 1621        pslldq	xmm7, 12
 1622        psrldq	xmm4, 4
 1623        aesenc	xmm12, [r15+144]
 1624        pxor	xmm2, xmm7
 1625        movdqa	xmm5, xmm2
 1626        movdqa	xmm1, xmm2
 1627        movdqa	xmm0, xmm2
 1628        aesenc	xmm13, [r15+144]
 1629        psrld	xmm5, 1
 1630        psrld	xmm1, 2
 1631        psrld	xmm0, 7
 1632        aesenc	xmm14, [r15+144]
 1633        pxor	xmm5, xmm1
 1634        pxor	xmm5, xmm0
 1635        aesenc	xmm15, [r15+144]
 1636        pxor	xmm5, xmm4
 1637        pxor	xmm2, xmm5
 1638        pxor	xmm2, xmm3
 1639        cmp	r10d, 11
 1640        movdqa	xmm7, OWORD PTR [r15+160]
 1641        jl	L_AES_GCM_encrypt_aesni_aesenc_128_ghash_avx_done
 1642        aesenc	xmm8, xmm7
 1643        aesenc	xmm9, xmm7
 1644        aesenc	xmm10, xmm7
 1645        aesenc	xmm11, xmm7
 1646        aesenc	xmm12, xmm7
 1647        aesenc	xmm13, xmm7
 1648        aesenc	xmm14, xmm7
 1649        aesenc	xmm15, xmm7
 1650        movdqa	xmm7, OWORD PTR [r15+176]
 1651        aesenc	xmm8, xmm7
 1652        aesenc	xmm9, xmm7
 1653        aesenc	xmm10, xmm7
 1654        aesenc	xmm11, xmm7
 1655        aesenc	xmm12, xmm7
 1656        aesenc	xmm13, xmm7
 1657        aesenc	xmm14, xmm7
 1658        aesenc	xmm15, xmm7
 1659        cmp	r10d, 13
 1660        movdqa	xmm7, OWORD PTR [r15+192]
 1661        jl	L_AES_GCM_encrypt_aesni_aesenc_128_ghash_avx_done
 1662        aesenc	xmm8, xmm7
 1663        aesenc	xmm9, xmm7
 1664        aesenc	xmm10, xmm7
 1665        aesenc	xmm11, xmm7
 1666        aesenc	xmm12, xmm7
 1667        aesenc	xmm13, xmm7
 1668        aesenc	xmm14, xmm7
 1669        aesenc	xmm15, xmm7
 1670        movdqa	xmm7, OWORD PTR [r15+208]
 1671        aesenc	xmm8, xmm7
 1672        aesenc	xmm9, xmm7
 1673        aesenc	xmm10, xmm7
 1674        aesenc	xmm11, xmm7
 1675        aesenc	xmm12, xmm7
 1676        aesenc	xmm13, xmm7
 1677        aesenc	xmm14, xmm7
 1678        aesenc	xmm15, xmm7
 1679        movdqa	xmm7, OWORD PTR [r15+224]
 1680L_AES_GCM_encrypt_aesni_aesenc_128_ghash_avx_done:
 1681        aesenclast	xmm8, xmm7
 1682        aesenclast	xmm9, xmm7
 1683        movdqu	xmm0, OWORD PTR [rcx]
 1684        movdqu	xmm1, OWORD PTR [rcx+16]
 1685        pxor	xmm8, xmm0
 1686        pxor	xmm9, xmm1
 1687        movdqu	OWORD PTR [rdx], xmm8
 1688        movdqu	OWORD PTR [rdx+16], xmm9
 1689        aesenclast	xmm10, xmm7
 1690        aesenclast	xmm11, xmm7
 1691        movdqu	xmm0, OWORD PTR [rcx+32]
 1692        movdqu	xmm1, OWORD PTR [rcx+48]
 1693        pxor	xmm10, xmm0
 1694        pxor	xmm11, xmm1
 1695        movdqu	OWORD PTR [rdx+32], xmm10
 1696        movdqu	OWORD PTR [rdx+48], xmm11
 1697        aesenclast	xmm12, xmm7
 1698        aesenclast	xmm13, xmm7
 1699        movdqu	xmm0, OWORD PTR [rcx+64]
 1700        movdqu	xmm1, OWORD PTR [rcx+80]
 1701        pxor	xmm12, xmm0
 1702        pxor	xmm13, xmm1
 1703        movdqu	OWORD PTR [rdx+64], xmm12
 1704        movdqu	OWORD PTR [rdx+80], xmm13
 1705        aesenclast	xmm14, xmm7
 1706        aesenclast	xmm15, xmm7
 1707        movdqu	xmm0, OWORD PTR [rcx+96]
 1708        movdqu	xmm1, OWORD PTR [rcx+112]
 1709        pxor	xmm14, xmm0
 1710        pxor	xmm15, xmm1
 1711        movdqu	OWORD PTR [rdx+96], xmm14
 1712        movdqu	OWORD PTR [rdx+112], xmm15
 1713        add	ebx, 128
 1714        cmp	ebx, r13d
 1715        jl	L_AES_GCM_encrypt_aesni_ghash_128
 1716L_AES_GCM_encrypt_aesni_end_128:
 1717        movdqa	xmm4, OWORD PTR L_aes_gcm_bswap_mask
 1718        pshufb	xmm8, xmm4
 1719        pshufb	xmm9, xmm4
 1720        pshufb	xmm10, xmm4
 1721        pshufb	xmm11, xmm4
 1722        pxor	xmm8, xmm2
 1723        pshufb	xmm12, xmm4
 1724        pshufb	xmm13, xmm4
 1725        pshufb	xmm14, xmm4
 1726        pshufb	xmm15, xmm4
 1727        movdqu	xmm7, OWORD PTR [rsp+112]
 1728        pshufd	xmm1, xmm8, 78
 1729        pshufd	xmm2, xmm7, 78
 1730        movdqa	xmm3, xmm7
 1731        movdqa	xmm0, xmm7
 1732        pclmulqdq	xmm3, xmm8, 17
 1733        pclmulqdq	xmm0, xmm8, 0
 1734        pxor	xmm1, xmm8
 1735        pxor	xmm2, xmm7
 1736        pclmulqdq	xmm1, xmm2, 0
 1737        pxor	xmm1, xmm0
 1738        pxor	xmm1, xmm3
 1739        movdqa	xmm2, xmm1
 1740        movdqa	xmm4, xmm0
 1741        movdqa	xmm6, xmm3
 1742        pslldq	xmm2, 8
 1743        psrldq	xmm1, 8
 1744        pxor	xmm4, xmm2
 1745        pxor	xmm6, xmm1
 1746        movdqu	xmm7, OWORD PTR [rsp+96]
 1747        pshufd	xmm1, xmm9, 78
 1748        pshufd	xmm2, xmm7, 78
 1749        movdqa	xmm3, xmm7
 1750        movdqa	xmm0, xmm7
 1751        pclmulqdq	xmm3, xmm9, 17
 1752        pclmulqdq	xmm0, xmm9, 0
 1753        pxor	xmm1, xmm9
 1754        pxor	xmm2, xmm7
 1755        pclmulqdq	xmm1, xmm2, 0
 1756        pxor	xmm1, xmm0
 1757        pxor	xmm1, xmm3
 1758        movdqa	xmm2, xmm1
 1759        pxor	xmm4, xmm0
 1760        pxor	xmm6, xmm3
 1761        pslldq	xmm2, 8
 1762        psrldq	xmm1, 8
 1763        pxor	xmm4, xmm2
 1764        pxor	xmm6, xmm1
 1765        movdqu	xmm7, OWORD PTR [rsp+80]
 1766        pshufd	xmm1, xmm10, 78
 1767        pshufd	xmm2, xmm7, 78
 1768        movdqa	xmm3, xmm7
 1769        movdqa	xmm0, xmm7
 1770        pclmulqdq	xmm3, xmm10, 17
 1771        pclmulqdq	xmm0, xmm10, 0
 1772        pxor	xmm1, xmm10
 1773        pxor	xmm2, xmm7
 1774        pclmulqdq	xmm1, xmm2, 0
 1775        pxor	xmm1, xmm0
 1776        pxor	xmm1, xmm3
 1777        movdqa	xmm2, xmm1
 1778        pxor	xmm4, xmm0
 1779        pxor	xmm6, xmm3
 1780        pslldq	xmm2, 8
 1781        psrldq	xmm1, 8
 1782        pxor	xmm4, xmm2
 1783        pxor	xmm6, xmm1
 1784        movdqu	xmm7, OWORD PTR [rsp+64]
 1785        pshufd	xmm1, xmm11, 78
 1786        pshufd	xmm2, xmm7, 78
 1787        movdqa	xmm3, xmm7
 1788        movdqa	xmm0, xmm7
 1789        pclmulqdq	xmm3, xmm11, 17
 1790        pclmulqdq	xmm0, xmm11, 0
 1791        pxor	xmm1, xmm11
 1792        pxor	xmm2, xmm7
 1793        pclmulqdq	xmm1, xmm2, 0
 1794        pxor	xmm1, xmm0
 1795        pxor	xmm1, xmm3
 1796        movdqa	xmm2, xmm1
 1797        pxor	xmm4, xmm0
 1798        pxor	xmm6, xmm3
 1799        pslldq	xmm2, 8
 1800        psrldq	xmm1, 8
 1801        pxor	xmm4, xmm2
 1802        pxor	xmm6, xmm1
 1803        movdqu	xmm7, OWORD PTR [rsp+48]
 1804        pshufd	xmm1, xmm12, 78
 1805        pshufd	xmm2, xmm7, 78
 1806        movdqa	xmm3, xmm7
 1807        movdqa	xmm0, xmm7
 1808        pclmulqdq	xmm3, xmm12, 17
 1809        pclmulqdq	xmm0, xmm12, 0
 1810        pxor	xmm1, xmm12
 1811        pxor	xmm2, xmm7
 1812        pclmulqdq	xmm1, xmm2, 0
 1813        pxor	xmm1, xmm0
 1814        pxor	xmm1, xmm3
 1815        movdqa	xmm2, xmm1
 1816        pxor	xmm4, xmm0
 1817        pxor	xmm6, xmm3
 1818        pslldq	xmm2, 8
 1819        psrldq	xmm1, 8
 1820        pxor	xmm4, xmm2
 1821        pxor	xmm6, xmm1
 1822        movdqu	xmm7, OWORD PTR [rsp+32]
 1823        pshufd	xmm1, xmm13, 78
 1824        pshufd	xmm2, xmm7, 78
 1825        movdqa	xmm3, xmm7
 1826        movdqa	xmm0, xmm7
 1827        pclmulqdq	xmm3, xmm13, 17
 1828        pclmulqdq	xmm0, xmm13, 0
 1829        pxor	xmm1, xmm13
 1830        pxor	xmm2, xmm7
 1831        pclmulqdq	xmm1, xmm2, 0
 1832        pxor	xmm1, xmm0
 1833        pxor	xmm1, xmm3
 1834        movdqa	xmm2, xmm1
 1835        pxor	xmm4, xmm0
 1836        pxor	xmm6, xmm3
 1837        pslldq	xmm2, 8
 1838        psrldq	xmm1, 8
 1839        pxor	xmm4, xmm2
 1840        pxor	xmm6, xmm1
 1841        movdqu	xmm7, OWORD PTR [rsp+16]
 1842        pshufd	xmm1, xmm14, 78
 1843        pshufd	xmm2, xmm7, 78
 1844        movdqa	xmm3, xmm7
 1845        movdqa	xmm0, xmm7
 1846        pclmulqdq	xmm3, xmm14, 17
 1847        pclmulqdq	xmm0, xmm14, 0
 1848        pxor	xmm1, xmm14
 1849        pxor	xmm2, xmm7
 1850        pclmulqdq	xmm1, xmm2, 0
 1851        pxor	xmm1, xmm0
 1852        pxor	xmm1, xmm3
 1853        movdqa	xmm2, xmm1
 1854        pxor	xmm4, xmm0
 1855        pxor	xmm6, xmm3
 1856        pslldq	xmm2, 8
 1857        psrldq	xmm1, 8
 1858        pxor	xmm4, xmm2
 1859        pxor	xmm6, xmm1
 1860        movdqu	xmm7, OWORD PTR [rsp]
 1861        pshufd	xmm1, xmm15, 78
 1862        pshufd	xmm2, xmm7, 78
 1863        movdqa	xmm3, xmm7
 1864        movdqa	xmm0, xmm7
 1865        pclmulqdq	xmm3, xmm15, 17
 1866        pclmulqdq	xmm0, xmm15, 0
 1867        pxor	xmm1, xmm15
 1868        pxor	xmm2, xmm7
 1869        pclmulqdq	xmm1, xmm2, 0
 1870        pxor	xmm1, xmm0
 1871        pxor	xmm1, xmm3
 1872        movdqa	xmm2, xmm1
 1873        pxor	xmm4, xmm0
 1874        pxor	xmm6, xmm3
 1875        pslldq	xmm2, 8
 1876        psrldq	xmm1, 8
 1877        pxor	xmm4, xmm2
 1878        pxor	xmm6, xmm1
 1879        movdqa	xmm0, xmm4
 1880        movdqa	xmm1, xmm4
 1881        movdqa	xmm2, xmm4
 1882        pslld	xmm0, 31
 1883        pslld	xmm1, 30
 1884        pslld	xmm2, 25
 1885        pxor	xmm0, xmm1
 1886        pxor	xmm0, xmm2
 1887        movdqa	xmm1, xmm0
 1888        psrldq	xmm1, 4
 1889        pslldq	xmm0, 12
 1890        pxor	xmm4, xmm0
 1891        movdqa	xmm2, xmm4
 1892        movdqa	xmm3, xmm4
 1893        movdqa	xmm0, xmm4
 1894        psrld	xmm2, 1
 1895        psrld	xmm3, 2
 1896        psrld	xmm0, 7
 1897        pxor	xmm2, xmm3
 1898        pxor	xmm2, xmm0
 1899        pxor	xmm2, xmm1
 1900        pxor	xmm2, xmm4
 1901        pxor	xmm6, xmm2
 1902        movdqu	xmm5, OWORD PTR [rsp]
 1903L_AES_GCM_encrypt_aesni_done_128:
 1904        mov	edx, r9d
 1905        cmp	ebx, edx
 1906        jge	L_AES_GCM_encrypt_aesni_done_enc
 1907        mov	r13d, r9d
 1908        and	r13d, 4294967280
 1909        cmp	ebx, r13d
 1910        jge	L_AES_GCM_encrypt_aesni_last_block_done
 1911        lea	rcx, QWORD PTR [rdi+rbx]
 1912        lea	rdx, QWORD PTR [rsi+rbx]
 1913        movdqu	xmm8, OWORD PTR [rsp+128]
 1914        movdqa	xmm9, xmm8
 1915        pshufb	xmm8, OWORD PTR L_aes_gcm_bswap_epi64
 1916        paddd	xmm9, OWORD PTR L_aes_gcm_one
 1917        pxor	xmm8, [r15]
 1918        movdqu	OWORD PTR [rsp+128], xmm9
 1919        aesenc	xmm8, [r15+16]
 1920        aesenc	xmm8, [r15+32]
 1921        aesenc	xmm8, [r15+48]
 1922        aesenc	xmm8, [r15+64]
 1923        aesenc	xmm8, [r15+80]
 1924        aesenc	xmm8, [r15+96]
 1925        aesenc	xmm8, [r15+112]
 1926        aesenc	xmm8, [r15+128]
 1927        aesenc	xmm8, [r15+144]
 1928        cmp	r10d, 11
 1929        movdqa	xmm9, OWORD PTR [r15+160]
 1930        jl	L_AES_GCM_encrypt_aesni_aesenc_block_aesenc_avx_last
 1931        aesenc	xmm8, xmm9
 1932        aesenc	xmm8, [r15+176]
 1933        cmp	r10d, 13
 1934        movdqa	xmm9, OWORD PTR [r15+192]
 1935        jl	L_AES_GCM_encrypt_aesni_aesenc_block_aesenc_avx_last
 1936        aesenc	xmm8, xmm9
 1937        aesenc	xmm8, [r15+208]
 1938        movdqa	xmm9, OWORD PTR [r15+224]
 1939L_AES_GCM_encrypt_aesni_aesenc_block_aesenc_avx_last:
 1940        aesenclast	xmm8, xmm9
 1941        movdqu	xmm9, OWORD PTR [rcx]
 1942        pxor	xmm8, xmm9
 1943        movdqu	OWORD PTR [rdx], xmm8
 1944        pshufb	xmm8, OWORD PTR L_aes_gcm_bswap_mask
 1945        pxor	xmm6, xmm8
 1946        add	ebx, 16
 1947        cmp	ebx, r13d
 1948        jge	L_AES_GCM_encrypt_aesni_last_block_ghash
 1949L_AES_GCM_encrypt_aesni_last_block_start:
 1950        lea	rcx, QWORD PTR [rdi+rbx]
 1951        lea	rdx, QWORD PTR [rsi+rbx]
 1952        movdqu	xmm8, OWORD PTR [rsp+128]
 1953        movdqa	xmm9, xmm8
 1954        pshufb	xmm8, OWORD PTR L_aes_gcm_bswap_epi64
 1955        paddd	xmm9, OWORD PTR L_aes_gcm_one
 1956        pxor	xmm8, [r15]
 1957        movdqu	OWORD PTR [rsp+128], xmm9
 1958        movdqa	xmm10, xmm6
 1959        pclmulqdq	xmm10, xmm5, 16
 1960        aesenc	xmm8, [r15+16]
 1961        aesenc	xmm8, [r15+32]
 1962        movdqa	xmm11, xmm6
 1963        pclmulqdq	xmm11, xmm5, 1
 1964        aesenc	xmm8, [r15+48]
 1965        aesenc	xmm8, [r15+64]
 1966        movdqa	xmm12, xmm6
 1967        pclmulqdq	xmm12, xmm5, 0
 1968        aesenc	xmm8, [r15+80]
 1969        movdqa	xmm1, xmm6
 1970        pclmulqdq	xmm1, xmm5, 17
 1971        aesenc	xmm8, [r15+96]
 1972        pxor	xmm10, xmm11
 1973        movdqa	xmm2, xmm10
 1974        psrldq	xmm10, 8
 1975        pslldq	xmm2, 8
 1976        aesenc	xmm8, [r15+112]
 1977        movdqa	xmm3, xmm1
 1978        pxor	xmm2, xmm12
 1979        pxor	xmm3, xmm10
 1980        movdqa	xmm0, OWORD PTR L_aes_gcm_mod2_128
 1981        movdqa	xmm11, xmm2
 1982        pclmulqdq	xmm11, xmm0, 16
 1983        aesenc	xmm8, [r15+128]
 1984        pshufd	xmm10, xmm2, 78
 1985        pxor	xmm10, xmm11
 1986        movdqa	xmm11, xmm10
 1987        pclmulqdq	xmm11, xmm0, 16
 1988        aesenc	xmm8, [r15+144]
 1989        pshufd	xmm6, xmm10, 78
 1990        pxor	xmm6, xmm11
 1991        pxor	xmm6, xmm3
 1992        cmp	r10d, 11
 1993        movdqa	xmm9, OWORD PTR [r15+160]
 1994        jl	L_AES_GCM_encrypt_aesni_aesenc_gfmul_last
 1995        aesenc	xmm8, xmm9
 1996        aesenc	xmm8, [r15+176]
 1997        cmp	r10d, 13
 1998        movdqa	xmm9, OWORD PTR [r15+192]
 1999        jl	L_AES_GCM_encrypt_aesni_aesenc_gfmul_last
 2000        aesenc	xmm8, xmm9
 2001        aesenc	xmm8, [r15+208]
 2002        movdqa	xmm9, OWORD PTR [r15+224]
 2003L_AES_GCM_encrypt_aesni_aesenc_gfmul_last:
 2004        aesenclast	xmm8, xmm9
 2005        movdqu	xmm9, OWORD PTR [rcx]
 2006        pxor	xmm8, xmm9
 2007        movdqu	OWORD PTR [rdx], xmm8
 2008        pshufb	xmm8, OWORD PTR L_aes_gcm_bswap_mask
 2009        pxor	xmm6, xmm8
 2010        add	ebx, 16
 2011        cmp	ebx, r13d
 2012        jl	L_AES_GCM_encrypt_aesni_last_block_start
 2013L_AES_GCM_encrypt_aesni_last_block_ghash:
 2014        pshufd	xmm9, xmm5, 78
 2015        pshufd	xmm10, xmm6, 78
 2016        movdqa	xmm11, xmm6
 2017        movdqa	xmm8, xmm6
 2018        pclmulqdq	xmm11, xmm5, 17
 2019        pclmulqdq	xmm8, xmm5, 0
 2020        pxor	xmm9, xmm5
 2021        pxor	xmm10, xmm6
 2022        pclmulqdq	xmm9, xmm10, 0
 2023        pxor	xmm9, xmm8
 2024        pxor	xmm9, xmm11
 2025        movdqa	xmm10, xmm9
 2026        movdqa	xmm6, xmm11
 2027        pslldq	xmm10, 8
 2028        psrldq	xmm9, 8
 2029        pxor	xmm8, xmm10
 2030        pxor	xmm6, xmm9
 2031        movdqa	xmm12, xmm8
 2032        movdqa	xmm13, xmm8
 2033        movdqa	xmm14, xmm8
 2034        pslld	xmm12, 31
 2035        pslld	xmm13, 30
 2036        pslld	xmm14, 25
 2037        pxor	xmm12, xmm13
 2038        pxor	xmm12, xmm14
 2039        movdqa	xmm13, xmm12
 2040        psrldq	xmm13, 4
 2041        pslldq	xmm12, 12
 2042        pxor	xmm8, xmm12
 2043        movdqa	xmm14, xmm8
 2044        movdqa	xmm10, xmm8
 2045        movdqa	xmm9, xmm8
 2046        psrld	xmm14, 1
 2047        psrld	xmm10, 2
 2048        psrld	xmm9, 7
 2049        pxor	xmm14, xmm10
 2050        pxor	xmm14, xmm9
 2051        pxor	xmm14, xmm13
 2052        pxor	xmm14, xmm8
 2053        pxor	xmm6, xmm14
 2054L_AES_GCM_encrypt_aesni_last_block_done:
 2055        mov	ecx, r9d
 2056        mov	edx, ecx
 2057        and	ecx, 15
 2058        jz	L_AES_GCM_encrypt_aesni_aesenc_last15_enc_avx_done
 2059        movdqu	xmm4, OWORD PTR [rsp+128]
 2060        pshufb	xmm4, OWORD PTR L_aes_gcm_bswap_epi64
 2061        pxor	xmm4, [r15]
 2062        aesenc	xmm4, [r15+16]
 2063        aesenc	xmm4, [r15+32]
 2064        aesenc	xmm4, [r15+48]
 2065        aesenc	xmm4, [r15+64]
 2066        aesenc	xmm4, [r15+80]
 2067        aesenc	xmm4, [r15+96]
 2068        aesenc	xmm4, [r15+112]
 2069        aesenc	xmm4, [r15+128]
 2070        aesenc	xmm4, [r15+144]
 2071        cmp	r10d, 11
 2072        movdqa	xmm9, OWORD PTR [r15+160]
 2073        jl	L_AES_GCM_encrypt_aesni_aesenc_last15_enc_avx_aesenc_avx_last
 2074        aesenc	xmm4, xmm9
 2075        aesenc	xmm4, [r15+176]
 2076        cmp	r10d, 13
 2077        movdqa	xmm9, OWORD PTR [r15+192]
 2078        jl	L_AES_GCM_encrypt_aesni_aesenc_last15_enc_avx_aesenc_avx_last
 2079        aesenc	xmm4, xmm9
 2080        aesenc	xmm4, [r15+208]
 2081        movdqa	xmm9, OWORD PTR [r15+224]
 2082L_AES_GCM_encrypt_aesni_aesenc_last15_enc_avx_aesenc_avx_last:
 2083        aesenclast	xmm4, xmm9
 2084        sub	rsp, 16
 2085        xor	ecx, ecx
 2086        movdqu	OWORD PTR [rsp], xmm4
 2087L_AES_GCM_encrypt_aesni_aesenc_last15_enc_avx_loop:
 2088        movzx	r13d, BYTE PTR [rdi+rbx]
 2089        xor	r13b, BYTE PTR [rsp+rcx]
 2090        mov	BYTE PTR [rsi+rbx], r13b
 2091        mov	BYTE PTR [rsp+rcx], r13b
 2092        inc	ebx
 2093        inc	ecx
 2094        cmp	ebx, edx
 2095        jl	L_AES_GCM_encrypt_aesni_aesenc_last15_enc_avx_loop
 2096        xor	r13, r13
 2097        cmp	ecx, 16
 2098        je	L_AES_GCM_encrypt_aesni_aesenc_last15_enc_avx_finish_enc
 2099L_AES_GCM_encrypt_aesni_aesenc_last15_enc_avx_byte_loop:
 2100        mov	BYTE PTR [rsp+rcx], r13b
 2101        inc	ecx
 2102        cmp	ecx, 16
 2103        jl	L_AES_GCM_encrypt_aesni_aesenc_last15_enc_avx_byte_loop
 2104L_AES_GCM_encrypt_aesni_aesenc_last15_enc_avx_finish_enc:
 2105        movdqu	xmm4, OWORD PTR [rsp]
 2106        add	rsp, 16
 2107        pshufb	xmm4, OWORD PTR L_aes_gcm_bswap_mask
 2108        pxor	xmm6, xmm4
 2109        pshufd	xmm9, xmm5, 78
 2110        pshufd	xmm10, xmm6, 78
 2111        movdqa	xmm11, xmm6
 2112        movdqa	xmm8, xmm6
 2113        pclmulqdq	xmm11, xmm5, 17
 2114        pclmulqdq	xmm8, xmm5, 0
 2115        pxor	xmm9, xmm5
 2116        pxor	xmm10, xmm6
 2117        pclmulqdq	xmm9, xmm10, 0
 2118        pxor	xmm9, xmm8
 2119        pxor	xmm9, xmm11
 2120        movdqa	xmm10, xmm9
 2121        movdqa	xmm6, xmm11
 2122        pslldq	xmm10, 8
 2123        psrldq	xmm9, 8
 2124        pxor	xmm8, xmm10
 2125        pxor	xmm6, xmm9
 2126        movdqa	xmm12, xmm8
 2127        movdqa	xmm13, xmm8
 2128        movdqa	xmm14, xmm8
 2129        pslld	xmm12, 31
 2130        pslld	xmm13, 30
 2131        pslld	xmm14, 25
 2132        pxor	xmm12, xmm13
 2133        pxor	xmm12, xmm14
 2134        movdqa	xmm13, xmm12
 2135        psrldq	xmm13, 4
 2136        pslldq	xmm12, 12
 2137        pxor	xmm8, xmm12
 2138        movdqa	xmm14, xmm8
 2139        movdqa	xmm10, xmm8
 2140        movdqa	xmm9, xmm8
 2141        psrld	xmm14, 1
 2142        psrld	xmm10, 2
 2143        psrld	xmm9, 7
 2144        pxor	xmm14, xmm10
 2145        pxor	xmm14, xmm9
 2146        pxor	xmm14, xmm13
 2147        pxor	xmm14, xmm8
 2148        pxor	xmm6, xmm14
 2149L_AES_GCM_encrypt_aesni_aesenc_last15_enc_avx_done:
 2150L_AES_GCM_encrypt_aesni_done_enc:
 2151        mov	edx, r9d
 2152        mov	ecx, r11d
 2153        shl	rdx, 3
 2154        shl	rcx, 3
 2155        pinsrq	xmm0, rdx, 0
 2156        pinsrq	xmm0, rcx, 1
 2157        pxor	xmm6, xmm0
 2158        pshufd	xmm9, xmm5, 78
 2159        pshufd	xmm10, xmm6, 78
 2160        movdqa	xmm11, xmm6
 2161        movdqa	xmm8, xmm6
 2162        pclmulqdq	xmm11, xmm5, 17
 2163        pclmulqdq	xmm8, xmm5, 0
 2164        pxor	xmm9, xmm5
 2165        pxor	xmm10, xmm6
 2166        pclmulqdq	xmm9, xmm10, 0
 2167        pxor	xmm9, xmm8
 2168        pxor	xmm9, xmm11
 2169        movdqa	xmm10, xmm9
 2170        movdqa	xmm6, xmm11
 2171        pslldq	xmm10, 8
 2172        psrldq	xmm9, 8
 2173        pxor	xmm8, xmm10
 2174        pxor	xmm6, xmm9
 2175        movdqa	xmm12, xmm8
 2176        movdqa	xmm13, xmm8
 2177        movdqa	xmm14, xmm8
 2178        pslld	xmm12, 31
 2179        pslld	xmm13, 30
 2180        pslld	xmm14, 25
 2181        pxor	xmm12, xmm13
 2182        pxor	xmm12, xmm14
 2183        movdqa	xmm13, xmm12
 2184        psrldq	xmm13, 4
 2185        pslldq	xmm12, 12
 2186        pxor	xmm8, xmm12
 2187        movdqa	xmm14, xmm8
 2188        movdqa	xmm10, xmm8
 2189        movdqa	xmm9, xmm8
 2190        psrld	xmm14, 1
 2191        psrld	xmm10, 2
 2192        psrld	xmm9, 7
 2193        pxor	xmm14, xmm10
 2194        pxor	xmm14, xmm9
 2195        pxor	xmm14, xmm13
 2196        pxor	xmm14, xmm8
 2197        pxor	xmm6, xmm14
 2198        pshufb	xmm6, OWORD PTR L_aes_gcm_bswap_mask
 2199        movdqu	xmm0, OWORD PTR [rsp+144]
 2200        pxor	xmm0, xmm6
 2201        cmp	r14d, 16
 2202        je	L_AES_GCM_encrypt_aesni_store_tag_16
 2203        xor	rcx, rcx
 2204        movdqu	OWORD PTR [rsp], xmm0
 2205L_AES_GCM_encrypt_aesni_store_tag_loop:
 2206        movzx	r13d, BYTE PTR [rsp+rcx]
 2207        mov	BYTE PTR [r8+rcx], r13b
 2208        inc	ecx
 2209        cmp	ecx, r14d
 2210        jne	L_AES_GCM_encrypt_aesni_store_tag_loop
 2211        jmp	L_AES_GCM_encrypt_aesni_store_tag_done
 2212L_AES_GCM_encrypt_aesni_store_tag_16:
 2213        movdqu	OWORD PTR [r8], xmm0
 2214L_AES_GCM_encrypt_aesni_store_tag_done:
 2215        movdqu	xmm6, OWORD PTR [rsp+160]
 2216        movdqu	xmm7, OWORD PTR [rsp+176]
 2217        movdqu	xmm8, OWORD PTR [rsp+192]
 2218        movdqu	xmm9, OWORD PTR [rsp+208]
 2219        movdqu	xmm10, OWORD PTR [rsp+224]
 2220        movdqu	xmm11, OWORD PTR [rsp+240]
 2221        movdqu	xmm12, OWORD PTR [rsp+256]
 2222        movdqu	xmm13, OWORD PTR [rsp+272]
 2223        movdqu	xmm14, OWORD PTR [rsp+288]
 2224        movdqu	xmm15, OWORD PTR [rsp+304]
 2225        add	rsp, 320
 2226        pop	r15
 2227        pop	r14
 2228        pop	rbx
 2229        pop	r12
 2230        pop	rsi
 2231        pop	rdi
 2232        pop	r13
 2233        ret
 2234AES_GCM_encrypt_aesni ENDP
 2235_TEXT ENDS
 2236_TEXT SEGMENT READONLY PARA
 2237AES_GCM_decrypt_aesni PROC
 2238        push	r13
 2239        push	rdi
 2240        push	rsi
 2241        push	r12
 2242        push	rbx
 2243        push	r14
 2244        push	r15
 2245        push	rbp
 2246        mov	rdi, rcx
 2247        mov	rsi, rdx
 2248        mov	r12, r8
 2249        mov	rax, r9
 2250        mov	r8, QWORD PTR [rsp+104]
 2251        mov	r9d, DWORD PTR [rsp+112]
 2252        mov	r11d, DWORD PTR [rsp+120]
 2253        mov	ebx, DWORD PTR [rsp+128]
 2254        mov	r14d, DWORD PTR [rsp+136]
 2255        mov	r15, QWORD PTR [rsp+144]
 2256        mov	r10d, DWORD PTR [rsp+152]
 2257        mov	rbp, QWORD PTR [rsp+160]
 2258        sub	rsp, 328
 2259        movdqu	OWORD PTR [rsp+168], xmm6
 2260        movdqu	OWORD PTR [rsp+184], xmm7
 2261        movdqu	OWORD PTR [rsp+200], xmm8
 2262        movdqu	OWORD PTR [rsp+216], xmm9
 2263        movdqu	OWORD PTR [rsp+232], xmm10
 2264        movdqu	OWORD PTR [rsp+248], xmm11
 2265        movdqu	OWORD PTR [rsp+264], xmm12
 2266        movdqu	OWORD PTR [rsp+280], xmm13
 2267        movdqu	OWORD PTR [rsp+296], xmm14
 2268        movdqu	OWORD PTR [rsp+312], xmm15
 2269        pxor	xmm4, xmm4
 2270        pxor	xmm6, xmm6
 2271        cmp	ebx, 12
 2272        mov	edx, ebx
 2273        jne	L_AES_GCM_decrypt_aesni_iv_not_12
 2274        ; # Calculate values when IV is 12 bytes
 2275        ; Set counter based on IV
 2276        mov	ecx, 16777216
 2277        pinsrq	xmm4, QWORD PTR [rax], 0
 2278        pinsrd	xmm4, DWORD PTR [rax+8], 2
 2279        pinsrd	xmm4, ecx, 3
 2280        ; H = Encrypt X(=0) and T = Encrypt counter
 2281        movdqa	xmm1, xmm4
 2282        movdqa	xmm5, OWORD PTR [r15]
 2283        pxor	xmm1, xmm5
 2284        movdqa	xmm7, OWORD PTR [r15+16]
 2285        aesenc	xmm5, xmm7
 2286        aesenc	xmm1, xmm7
 2287        movdqa	xmm7, OWORD PTR [r15+32]
 2288        aesenc	xmm5, xmm7
 2289        aesenc	xmm1, xmm7
 2290        movdqa	xmm7, OWORD PTR [r15+48]
 2291        aesenc	xmm5, xmm7
 2292        aesenc	xmm1, xmm7
 2293        movdqa	xmm7, OWORD PTR [r15+64]
 2294        aesenc	xmm5, xmm7
 2295        aesenc	xmm1, xmm7
 2296        movdqa	xmm7, OWORD PTR [r15+80]
 2297        aesenc	xmm5, xmm7
 2298        aesenc	xmm1, xmm7
 2299        movdqa	xmm7, OWORD PTR [r15+96]
 2300        aesenc	xmm5, xmm7
 2301        aesenc	xmm1, xmm7
 2302        movdqa	xmm7, OWORD PTR [r15+112]
 2303        aesenc	xmm5, xmm7
 2304        aesenc	xmm1, xmm7
 2305        movdqa	xmm7, OWORD PTR [r15+128]
 2306        aesenc	xmm5, xmm7
 2307        aesenc	xmm1, xmm7
 2308        movdqa	xmm7, OWORD PTR [r15+144]
 2309        aesenc	xmm5, xmm7
 2310        aesenc	xmm1, xmm7
 2311        cmp	r10d, 11
 2312        movdqa	xmm7, OWORD PTR [r15+160]
 2313        jl	L_AES_GCM_decrypt_aesni_calc_iv_12_last
 2314        aesenc	xmm5, xmm7
 2315        aesenc	xmm1, xmm7
 2316        movdqa	xmm7, OWORD PTR [r15+176]
 2317        aesenc	xmm5, xmm7
 2318        aesenc	xmm1, xmm7
 2319        cmp	r10d, 13
 2320        movdqa	xmm7, OWORD PTR [r15+192]
 2321        jl	L_AES_GCM_decrypt_aesni_calc_iv_12_last
 2322        aesenc	xmm5, xmm7
 2323        aesenc	xmm1, xmm7
 2324        movdqa	xmm7, OWORD PTR [r15+208]
 2325        aesenc	xmm5, xmm7
 2326        aesenc	xmm1, xmm7
 2327        movdqa	xmm7, OWORD PTR [r15+224]
 2328L_AES_GCM_decrypt_aesni_calc_iv_12_last:
 2329        aesenclast	xmm5, xmm7
 2330        aesenclast	xmm1, xmm7
 2331        pshufb	xmm5, OWORD PTR L_aes_gcm_bswap_mask
 2332        movdqu	OWORD PTR [rsp+144], xmm1
 2333        jmp	L_AES_GCM_decrypt_aesni_iv_done
 2334L_AES_GCM_decrypt_aesni_iv_not_12:
 2335        ; Calculate values when IV is not 12 bytes
 2336        ; H = Encrypt X(=0)
 2337        movdqa	xmm5, OWORD PTR [r15]
 2338        aesenc	xmm5, [r15+16]
 2339        aesenc	xmm5, [r15+32]
 2340        aesenc	xmm5, [r15+48]
 2341        aesenc	xmm5, [r15+64]
 2342        aesenc	xmm5, [r15+80]
 2343        aesenc	xmm5, [r15+96]
 2344        aesenc	xmm5, [r15+112]
 2345        aesenc	xmm5, [r15+128]
 2346        aesenc	xmm5, [r15+144]
 2347        cmp	r10d, 11
 2348        movdqa	xmm9, OWORD PTR [r15+160]
 2349        jl	L_AES_GCM_decrypt_aesni_calc_iv_1_aesenc_avx_last
 2350        aesenc	xmm5, xmm9
 2351        aesenc	xmm5, [r15+176]
 2352        cmp	r10d, 13
 2353        movdqa	xmm9, OWORD PTR [r15+192]
 2354        jl	L_AES_GCM_decrypt_aesni_calc_iv_1_aesenc_avx_last
 2355        aesenc	xmm5, xmm9
 2356        aesenc	xmm5, [r15+208]
 2357        movdqa	xmm9, OWORD PTR [r15+224]
 2358L_AES_GCM_decrypt_aesni_calc_iv_1_aesenc_avx_last:
 2359        aesenclast	xmm5, xmm9
 2360        pshufb	xmm5, OWORD PTR L_aes_gcm_bswap_mask
 2361        ; Calc counter
 2362        ; Initialization vector
 2363        cmp	edx, 0
 2364        mov	rcx, 0
 2365        je	L_AES_GCM_decrypt_aesni_calc_iv_done
 2366        cmp	edx, 16
 2367        jl	L_AES_GCM_decrypt_aesni_calc_iv_lt16
 2368        and	edx, 4294967280
 2369L_AES_GCM_decrypt_aesni_calc_iv_16_loop:
 2370        movdqu	xmm8, OWORD PTR [rax+rcx]
 2371        pshufb	xmm8, OWORD PTR L_aes_gcm_bswap_mask
 2372        pxor	xmm4, xmm8
 2373        pshufd	xmm1, xmm4, 78
 2374        pshufd	xmm2, xmm5, 78
 2375        movdqa	xmm3, xmm5
 2376        movdqa	xmm0, xmm5
 2377        pclmulqdq	xmm3, xmm4, 17
 2378        pclmulqdq	xmm0, xmm4, 0
 2379        pxor	xmm1, xmm4
 2380        pxor	xmm2, xmm5
 2381        pclmulqdq	xmm1, xmm2, 0
 2382        pxor	xmm1, xmm0
 2383        pxor	xmm1, xmm3
 2384        movdqa	xmm2, xmm1
 2385        movdqa	xmm7, xmm0
 2386        movdqa	xmm4, xmm3
 2387        pslldq	xmm2, 8
 2388        psrldq	xmm1, 8
 2389        pxor	xmm7, xmm2
 2390        pxor	xmm4, xmm1
 2391        movdqa	xmm0, xmm7
 2392        movdqa	xmm1, xmm4
 2393        psrld	xmm0, 31
 2394        psrld	xmm1, 31
 2395        pslld	xmm7, 1
 2396        pslld	xmm4, 1
 2397        movdqa	xmm2, xmm0
 2398        pslldq	xmm0, 4
 2399        psrldq	xmm2, 12
 2400        pslldq	xmm1, 4
 2401        por	xmm4, xmm2
 2402        por	xmm7, xmm0
 2403        por	xmm4, xmm1
 2404        movdqa	xmm0, xmm7
 2405        movdqa	xmm1, xmm7
 2406        movdqa	xmm2, xmm7
 2407        pslld	xmm0, 31
 2408        pslld	xmm1, 30
 2409        pslld	xmm2, 25
 2410        pxor	xmm0, xmm1
 2411        pxor	xmm0, xmm2
 2412        movdqa	xmm1, xmm0
 2413        psrldq	xmm1, 4
 2414        pslldq	xmm0, 12
 2415        pxor	xmm7, xmm0
 2416        movdqa	xmm2, xmm7
 2417        movdqa	xmm3, xmm7
 2418        movdqa	xmm0, xmm7
 2419        psrld	xmm2, 1
 2420        psrld	xmm3, 2
 2421        psrld	xmm0, 7
 2422        pxor	xmm2, xmm3
 2423        pxor	xmm2, xmm0
 2424        pxor	xmm2, xmm1
 2425        pxor	xmm2, xmm7
 2426        pxor	xmm4, xmm2
 2427        add	ecx, 16
 2428        cmp	ecx, edx
 2429        jl	L_AES_GCM_decrypt_aesni_calc_iv_16_loop
 2430        mov	edx, ebx
 2431        cmp	ecx, edx
 2432        je	L_AES_GCM_decrypt_aesni_calc_iv_done
 2433L_AES_GCM_decrypt_aesni_calc_iv_lt16:
 2434        sub	rsp, 16
 2435        pxor	xmm8, xmm8
 2436        xor	ebx, ebx
 2437        movdqu	OWORD PTR [rsp], xmm8
 2438L_AES_GCM_decrypt_aesni_calc_iv_loop:
 2439        movzx	r13d, BYTE PTR [rax+rcx]
 2440        mov	BYTE PTR [rsp+rbx], r13b
 2441        inc	ecx
 2442        inc	ebx
 2443        cmp	ecx, edx
 2444        jl	L_AES_GCM_decrypt_aesni_calc_iv_loop
 2445        movdqu	xmm8, OWORD PTR [rsp]
 2446        add	rsp, 16
 2447        pshufb	xmm8, OWORD PTR L_aes_gcm_bswap_mask
 2448        pxor	xmm4, xmm8
 2449        pshufd	xmm1, xmm4, 78
 2450        pshufd	xmm2, xmm5, 78
 2451        movdqa	xmm3, xmm5
 2452        movdqa	xmm0, xmm5
 2453        pclmulqdq	xmm3, xmm4, 17
 2454        pclmulqdq	xmm0, xmm4, 0
 2455        pxor	xmm1, xmm4
 2456        pxor	xmm2, xmm5
 2457        pclmulqdq	xmm1, xmm2, 0
 2458        pxor	xmm1, xmm0
 2459        pxor	xmm1, xmm3
 2460        movdqa	xmm2, xmm1
 2461        movdqa	xmm7, xmm0
 2462        movdqa	xmm4, xmm3
 2463        pslldq	xmm2, 8
 2464        psrldq	xmm1, 8
 2465        pxor	xmm7, xmm2
 2466        pxor	xmm4, xmm1
 2467        movdqa	xmm0, xmm7
 2468        movdqa	xmm1, xmm4
 2469        psrld	xmm0, 31
 2470        psrld	xmm1, 31
 2471        pslld	xmm7, 1
 2472        pslld	xmm4, 1
 2473        movdqa	xmm2, xmm0
 2474        pslldq	xmm0, 4
 2475        psrldq	xmm2, 12
 2476        pslldq	xmm1, 4
 2477        por	xmm4, xmm2
 2478        por	xmm7, xmm0
 2479        por	xmm4, xmm1
 2480        movdqa	xmm0, xmm7
 2481        movdqa	xmm1, xmm7
 2482        movdqa	xmm2, xmm7
 2483        pslld	xmm0, 31
 2484        pslld	xmm1, 30
 2485        pslld	xmm2, 25
 2486        pxor	xmm0, xmm1
 2487        pxor	xmm0, xmm2
 2488        movdqa	xmm1, xmm0
 2489        psrldq	xmm1, 4
 2490        pslldq	xmm0, 12
 2491        pxor	xmm7, xmm0
 2492        movdqa	xmm2, xmm7
 2493        movdqa	xmm3, xmm7
 2494        movdqa	xmm0, xmm7
 2495        psrld	xmm2, 1
 2496        psrld	xmm3, 2
 2497        psrld	xmm0, 7
 2498        pxor	xmm2, xmm3
 2499        pxor	xmm2, xmm0
 2500        pxor	xmm2, xmm1
 2501        pxor	xmm2, xmm7
 2502        pxor	xmm4, xmm2
 2503L_AES_GCM_decrypt_aesni_calc_iv_done:
 2504        ; T = Encrypt counter
 2505        pxor	xmm0, xmm0
 2506        shl	edx, 3
 2507        pinsrq	xmm0, rdx, 0
 2508        pxor	xmm4, xmm0
 2509        pshufd	xmm1, xmm4, 78
 2510        pshufd	xmm2, xmm5, 78
 2511        movdqa	xmm3, xmm5
 2512        movdqa	xmm0, xmm5
 2513        pclmulqdq	xmm3, xmm4, 17
 2514        pclmulqdq	xmm0, xmm4, 0
 2515        pxor	xmm1, xmm4
 2516        pxor	xmm2, xmm5
 2517        pclmulqdq	xmm1, xmm2, 0
 2518        pxor	xmm1, xmm0
 2519        pxor	xmm1, xmm3
 2520        movdqa	xmm2, xmm1
 2521        movdqa	xmm7, xmm0
 2522        movdqa	xmm4, xmm3
 2523        pslldq	xmm2, 8
 2524        psrldq	xmm1, 8
 2525        pxor	xmm7, xmm2
 2526        pxor	xmm4, xmm1
 2527        movdqa	xmm0, xmm7
 2528        movdqa	xmm1, xmm4
 2529        psrld	xmm0, 31
 2530        psrld	xmm1, 31
 2531        pslld	xmm7, 1
 2532        pslld	xmm4, 1
 2533        movdqa	xmm2, xmm0
 2534        pslldq	xmm0, 4
 2535        psrldq	xmm2, 12
 2536        pslldq	xmm1, 4
 2537        por	xmm4, xmm2
 2538        por	xmm7, xmm0
 2539        por	xmm4, xmm1
 2540        movdqa	xmm0, xmm7
 2541        movdqa	xmm1, xmm7
 2542        movdqa	xmm2, xmm7
 2543        pslld	xmm0, 31
 2544        pslld	xmm1, 30
 2545        pslld	xmm2, 25
 2546        pxor	xmm0, xmm1
 2547        pxor	xmm0, xmm2
 2548        movdqa	xmm1, xmm0
 2549        psrldq	xmm1, 4
 2550        pslldq	xmm0, 12
 2551        pxor	xmm7, xmm0
 2552        movdqa	xmm2, xmm7
 2553        movdqa	xmm3, xmm7
 2554        movdqa	xmm0, xmm7
 2555        psrld	xmm2, 1
 2556        psrld	xmm3, 2
 2557        psrld	xmm0, 7
 2558        pxor	xmm2, xmm3
 2559        pxor	xmm2, xmm0
 2560        pxor	xmm2, xmm1
 2561        pxor	xmm2, xmm7
 2562        pxor	xmm4, xmm2
 2563        pshufb	xmm4, OWORD PTR L_aes_gcm_bswap_mask
 2564        ;   Encrypt counter
 2565        movdqa	xmm8, OWORD PTR [r15]
 2566        pxor	xmm8, xmm4
 2567        aesenc	xmm8, [r15+16]
 2568        aesenc	xmm8, [r15+32]
 2569        aesenc	xmm8, [r15+48]
 2570        aesenc	xmm8, [r15+64]
 2571        aesenc	xmm8, [r15+80]
 2572        aesenc	xmm8, [r15+96]
 2573        aesenc	xmm8, [r15+112]
 2574        aesenc	xmm8, [r15+128]
 2575        aesenc	xmm8, [r15+144]
 2576        cmp	r10d, 11
 2577        movdqa	xmm9, OWORD PTR [r15+160]
 2578        jl	L_AES_GCM_decrypt_aesni_calc_iv_2_aesenc_avx_last
 2579        aesenc	xmm8, xmm9
 2580        aesenc	xmm8, [r15+176]
 2581        cmp	r10d, 13
 2582        movdqa	xmm9, OWORD PTR [r15+192]
 2583        jl	L_AES_GCM_decrypt_aesni_calc_iv_2_aesenc_avx_last
 2584        aesenc	xmm8, xmm9
 2585        aesenc	xmm8, [r15+208]
 2586        movdqa	xmm9, OWORD PTR [r15+224]
 2587L_AES_GCM_decrypt_aesni_calc_iv_2_aesenc_avx_last:
 2588        aesenclast	xmm8, xmm9
 2589        movdqu	OWORD PTR [rsp+144], xmm8
 2590L_AES_GCM_decrypt_aesni_iv_done:
 2591        ; Additional authentication data
 2592        mov	edx, r11d
 2593        cmp	edx, 0
 2594        je	L_AES_GCM_decrypt_aesni_calc_aad_done
 2595        xor	ecx, ecx
 2596        cmp	edx, 16
 2597        jl	L_AES_GCM_decrypt_aesni_calc_aad_lt16
 2598        and	edx, 4294967280
 2599L_AES_GCM_decrypt_aesni_calc_aad_16_loop:
 2600        movdqu	xmm8, OWORD PTR [r12+rcx]
 2601        pshufb	xmm8, OWORD PTR L_aes_gcm_bswap_mask
 2602        pxor	xmm6, xmm8
 2603        pshufd	xmm1, xmm6, 78
 2604        pshufd	xmm2, xmm5, 78
 2605        movdqa	xmm3, xmm5
 2606        movdqa	xmm0, xmm5
 2607        pclmulqdq	xmm3, xmm6, 17
 2608        pclmulqdq	xmm0, xmm6, 0
 2609        pxor	xmm1, xmm6
 2610        pxor	xmm2, xmm5
 2611        pclmulqdq	xmm1, xmm2, 0
 2612        pxor	xmm1, xmm0
 2613        pxor	xmm1, xmm3
 2614        movdqa	xmm2, xmm1
 2615        movdqa	xmm7, xmm0
 2616        movdqa	xmm6, xmm3
 2617        pslldq	xmm2, 8
 2618        psrldq	xmm1, 8
 2619        pxor	xmm7, xmm2
 2620        pxor	xmm6, xmm1
 2621        movdqa	xmm0, xmm7
 2622        movdqa	xmm1, xmm6
 2623        psrld	xmm0, 31
 2624        psrld	xmm1, 31
 2625        pslld	xmm7, 1
 2626        pslld	xmm6, 1
 2627        movdqa	xmm2, xmm0
 2628        pslldq	xmm0, 4
 2629        psrldq	xmm2, 12
 2630        pslldq	xmm1, 4
 2631        por	xmm6, xmm2
 2632        por	xmm7, xmm0
 2633        por	xmm6, xmm1
 2634        movdqa	xmm0, xmm7
 2635        movdqa	xmm1, xmm7
 2636        movdqa	xmm2, xmm7
 2637        pslld	xmm0, 31
 2638        pslld	xmm1, 30
 2639        pslld	xmm2, 25
 2640        pxor	xmm0, xmm1
 2641        pxor	xmm0, xmm2
 2642        movdqa	xmm1, xmm0
 2643        psrldq	xmm1, 4
 2644        pslldq	xmm0, 12
 2645        pxor	xmm7, xmm0
 2646        movdqa	xmm2, xmm7
 2647        movdqa	xmm3, xmm7
 2648        movdqa	xmm0, xmm7
 2649        psrld	xmm2, 1
 2650        psrld	xmm3, 2
 2651        psrld	xmm0, 7
 2652        pxor	xmm2, xmm3
 2653        pxor	xmm2, xmm0
 2654        pxor	xmm2, xmm1
 2655        pxor	xmm2, xmm7
 2656        pxor	xmm6, xmm2
 2657        add	ecx, 16
 2658        cmp	ecx, edx
 2659        jl	L_AES_GCM_decrypt_aesni_calc_aad_16_loop
 2660        mov	edx, r11d
 2661        cmp	ecx, edx
 2662        je	L_AES_GCM_decrypt_aesni_calc_aad_done
 2663L_AES_GCM_decrypt_aesni_calc_aad_lt16:
 2664        sub	rsp, 16
 2665        pxor	xmm8, xmm8
 2666        xor	ebx, ebx
 2667        movdqu	OWORD PTR [rsp], xmm8
 2668L_AES_GCM_decrypt_aesni_calc_aad_loop:
 2669        movzx	r13d, BYTE PTR [r12+rcx]
 2670        mov	BYTE PTR [rsp+rbx], r13b
 2671        inc	ecx
 2672        inc	ebx
 2673        cmp	ecx, edx
 2674        jl	L_AES_GCM_decrypt_aesni_calc_aad_loop
 2675        movdqu	xmm8, OWORD PTR [rsp]
 2676        add	rsp, 16
 2677        pshufb	xmm8, OWORD PTR L_aes_gcm_bswap_mask
 2678        pxor	xmm6, xmm8
 2679        pshufd	xmm1, xmm6, 78
 2680        pshufd	xmm2, xmm5, 78
 2681        movdqa	xmm3, xmm5
 2682        movdqa	xmm0, xmm5
 2683        pclmulqdq	xmm3, xmm6, 17
 2684        pclmulqdq	xmm0, xmm6, 0
 2685        pxor	xmm1, xmm6
 2686        pxor	xmm2, xmm5
 2687        pclmulqdq	xmm1, xmm2, 0
 2688        pxor	xmm1, xmm0
 2689        pxor	xmm1, xmm3
 2690        movdqa	xmm2, xmm1
 2691        movdqa	xmm7, xmm0
 2692        movdqa	xmm6, xmm3
 2693        pslldq	xmm2, 8
 2694        psrldq	xmm1, 8
 2695        pxor	xmm7, xmm2
 2696        pxor	xmm6, xmm1
 2697        movdqa	xmm0, xmm7
 2698        movdqa	xmm1, xmm6
 2699        psrld	xmm0, 31
 2700        psrld	xmm1, 31
 2701        pslld	xmm7, 1
 2702        pslld	xmm6, 1
 2703        movdqa	xmm2, xmm0
 2704        pslldq	xmm0, 4
 2705        psrldq	xmm2, 12
 2706        pslldq	xmm1, 4
 2707        por	xmm6, xmm2
 2708        por	xmm7, xmm0
 2709        por	xmm6, xmm1
 2710        movdqa	xmm0, xmm7
 2711        movdqa	xmm1, xmm7
 2712        movdqa	xmm2, xmm7
 2713        pslld	xmm0, 31
 2714        pslld	xmm1, 30
 2715        pslld	xmm2, 25
 2716        pxor	xmm0, xmm1
 2717        pxor	xmm0, xmm2
 2718        movdqa	xmm1, xmm0
 2719        psrldq	xmm1, 4
 2720        pslldq	xmm0, 12
 2721        pxor	xmm7, xmm0
 2722        movdqa	xmm2, xmm7
 2723        movdqa	xmm3, xmm7
 2724        movdqa	xmm0, xmm7
 2725        psrld	xmm2, 1
 2726        psrld	xmm3, 2
 2727        psrld	xmm0, 7
 2728        pxor	xmm2, xmm3
 2729        pxor	xmm2, xmm0
 2730        pxor	xmm2, xmm1
 2731        pxor	xmm2, xmm7
 2732        pxor	xmm6, xmm2
 2733L_AES_GCM_decrypt_aesni_calc_aad_done:
 2734        ; Calculate counter and H
 2735        pshufb	xmm4, OWORD PTR L_aes_gcm_bswap_epi64
 2736        movdqa	xmm9, xmm5
 2737        paddd	xmm4, OWORD PTR L_aes_gcm_one
 2738        movdqa	xmm8, xmm5
 2739        movdqu	OWORD PTR [rsp+128], xmm4
 2740        psrlq	xmm9, 63
 2741        psllq	xmm8, 1
 2742        pslldq	xmm9, 8
 2743        por	xmm8, xmm9
 2744        pshufd	xmm5, xmm5, 255
 2745        psrad	xmm5, 31
 2746        pand	xmm5, OWORD PTR L_aes_gcm_mod2_128
 2747        pxor	xmm5, xmm8
 2748        xor	ebx, ebx
 2749        cmp	r9d, 128
 2750        mov	r13d, r9d
 2751        jl	L_AES_GCM_decrypt_aesni_done_128
 2752        and	r13d, 4294967168
 2753        movdqa	xmm2, xmm6
 2754        ; H ^ 1
 2755        movdqu	OWORD PTR [rsp], xmm5
 2756        ; H ^ 2
 2757        pshufd	xmm9, xmm5, 78
 2758        pshufd	xmm10, xmm5, 78
 2759        movdqa	xmm11, xmm5
 2760        movdqa	xmm8, xmm5
 2761        pclmulqdq	xmm11, xmm5, 17
 2762        pclmulqdq	xmm8, xmm5, 0
 2763        pxor	xmm9, xmm5
 2764        pxor	xmm10, xmm5
 2765        pclmulqdq	xmm9, xmm10, 0
 2766        pxor	xmm9, xmm8
 2767        pxor	xmm9, xmm11
 2768        movdqa	xmm10, xmm9
 2769        movdqa	xmm0, xmm11
 2770        pslldq	xmm10, 8
 2771        psrldq	xmm9, 8
 2772        pxor	xmm8, xmm10
 2773        pxor	xmm0, xmm9
 2774        movdqa	xmm12, xmm8
 2775        movdqa	xmm13, xmm8
 2776        movdqa	xmm14, xmm8
 2777        pslld	xmm12, 31
 2778        pslld	xmm13, 30
 2779        pslld	xmm14, 25
 2780        pxor	xmm12, xmm13
 2781        pxor	xmm12, xmm14
 2782        movdqa	xmm13, xmm12
 2783        psrldq	xmm13, 4
 2784        pslldq	xmm12, 12
 2785        pxor	xmm8, xmm12
 2786        movdqa	xmm14, xmm8
 2787        movdqa	xmm10, xmm8
 2788        movdqa	xmm9, xmm8
 2789        psrld	xmm14, 1
 2790        psrld	xmm10, 2
 2791        psrld	xmm9, 7
 2792        pxor	xmm14, xmm10
 2793        pxor	xmm14, xmm9
 2794        pxor	xmm14, xmm13
 2795        pxor	xmm14, xmm8
 2796        pxor	xmm0, xmm14
 2797        movdqu	OWORD PTR [rsp+16], xmm0
 2798        ; H ^ 3
 2799        pshufd	xmm9, xmm5, 78
 2800        pshufd	xmm10, xmm0, 78
 2801        movdqa	xmm11, xmm0
 2802        movdqa	xmm8, xmm0
 2803        pclmulqdq	xmm11, xmm5, 17
 2804        pclmulqdq	xmm8, xmm5, 0
 2805        pxor	xmm9, xmm5
 2806        pxor	xmm10, xmm0
 2807        pclmulqdq	xmm9, xmm10, 0
 2808        pxor	xmm9, xmm8
 2809        pxor	xmm9, xmm11
 2810        movdqa	xmm10, xmm9
 2811        movdqa	xmm1, xmm11
 2812        pslldq	xmm10, 8
 2813        psrldq	xmm9, 8
 2814        pxor	xmm8, xmm10
 2815        pxor	xmm1, xmm9
 2816        movdqa	xmm12, xmm8
 2817        movdqa	xmm13, xmm8
 2818        movdqa	xmm14, xmm8
 2819        pslld	xmm12, 31
 2820        pslld	xmm13, 30
 2821        pslld	xmm14, 25
 2822        pxor	xmm12, xmm13
 2823        pxor	xmm12, xmm14
 2824        movdqa	xmm13, xmm12
 2825        psrldq	xmm13, 4
 2826        pslldq	xmm12, 12
 2827        pxor	xmm8, xmm12
 2828        movdqa	xmm14, xmm8
 2829        movdqa	xmm10, xmm8
 2830        movdqa	xmm9, xmm8
 2831        psrld	xmm14, 1
 2832        psrld	xmm10, 2
 2833        psrld	xmm9, 7
 2834        pxor	xmm14, xmm10
 2835        pxor	xmm14, xmm9
 2836        pxor	xmm14, xmm13
 2837        pxor	xmm14, xmm8
 2838        pxor	xmm1, xmm14
 2839        movdqu	OWORD PTR [rsp+32], xmm1
 2840        ; H ^ 4
 2841        pshufd	xmm9, xmm0, 78
 2842        pshufd	xmm10, xmm0, 78
 2843        movdqa	xmm11, xmm0
 2844        movdqa	xmm8, xmm0
 2845        pclmulqdq	xmm11, xmm0, 17
 2846        pclmulqdq	xmm8, xmm0, 0
 2847        pxor	xmm9, xmm0
 2848        pxor	xmm10, xmm0
 2849        pclmulqdq	xmm9, xmm10, 0
 2850        pxor	xmm9, xmm8
 2851        pxor	xmm9, xmm11
 2852        movdqa	xmm10, xmm9
 2853        movdqa	xmm3, xmm11
 2854        pslldq	xmm10, 8
 2855        psrldq	xmm9, 8
 2856        pxor	xmm8, xmm10
 2857        pxor	xmm3, xmm9
 2858        movdqa	xmm12, xmm8
 2859        movdqa	xmm13, xmm8
 2860        movdqa	xmm14, xmm8
 2861        pslld	xmm12, 31
 2862        pslld	xmm13, 30
 2863        pslld	xmm14, 25
 2864        pxor	xmm12, xmm13
 2865        pxor	xmm12, xmm14
 2866        movdqa	xmm13, xmm12
 2867        psrldq	xmm13, 4
 2868        pslldq	xmm12, 12
 2869        pxor	xmm8, xmm12
 2870        movdqa	xmm14, xmm8
 2871        movdqa	xmm10, xmm8
 2872        movdqa	xmm9, xmm8
 2873        psrld	xmm14, 1
 2874        psrld	xmm10, 2
 2875        psrld	xmm9, 7
 2876        pxor	xmm14, xmm10
 2877        pxor	xmm14, xmm9
 2878        pxor	xmm14, xmm13
 2879        pxor	xmm14, xmm8
 2880        pxor	xmm3, xmm14
 2881        movdqu	OWORD PTR [rsp+48], xmm3
 2882        ; H ^ 5
 2883        pshufd	xmm9, xmm0, 78
 2884        pshufd	xmm10, xmm1, 78
 2885        movdqa	xmm11, xmm1
 2886        movdqa	xmm8, xmm1
 2887        pclmulqdq	xmm11, xmm0, 17
 2888        pclmulqdq	xmm8, xmm0, 0
 2889        pxor	xmm9, xmm0
 2890        pxor	xmm10, xmm1
 2891        pclmulqdq	xmm9, xmm10, 0
 2892        pxor	xmm9, xmm8
 2893        pxor	xmm9, xmm11
 2894        movdqa	xmm10, xmm9
 2895        movdqa	xmm7, xmm11
 2896        pslldq	xmm10, 8
 2897        psrldq	xmm9, 8
 2898        pxor	xmm8, xmm10
 2899        pxor	xmm7, xmm9
 2900        movdqa	xmm12, xmm8
 2901        movdqa	xmm13, xmm8
 2902        movdqa	xmm14, xmm8
 2903        pslld	xmm12, 31
 2904        pslld	xmm13, 30
 2905        pslld	xmm14, 25
 2906        pxor	xmm12, xmm13
 2907        pxor	xmm12, xmm14
 2908        movdqa	xmm13, xmm12
 2909        psrldq	xmm13, 4
 2910        pslldq	xmm12, 12
 2911        pxor	xmm8, xmm12
 2912        movdqa	xmm14, xmm8
 2913        movdqa	xmm10, xmm8
 2914        movdqa	xmm9, xmm8
 2915        psrld	xmm14, 1
 2916        psrld	xmm10, 2
 2917        psrld	xmm9, 7
 2918        pxor	xmm14, xmm10
 2919        pxor	xmm14, xmm9
 2920        pxor	xmm14, xmm13
 2921        pxor	xmm14, xmm8
 2922        pxor	xmm7, xmm14
 2923        movdqu	OWORD PTR [rsp+64], xmm7
 2924        ; H ^ 6
 2925        pshufd	xmm9, xmm1, 78
 2926        pshufd	xmm10, xmm1, 78
 2927        movdqa	xmm11, xmm1
 2928        movdqa	xmm8, xmm1
 2929        pclmulqdq	xmm11, xmm1, 17
 2930        pclmulqdq	xmm8, xmm1, 0
 2931        pxor	xmm9, xmm1
 2932        pxor	xmm10, xmm1
 2933        pclmulqdq	xmm9, xmm10, 0
 2934        pxor	xmm9, xmm8
 2935        pxor	xmm9, xmm11
 2936        movdqa	xmm10, xmm9
 2937        movdqa	xmm7, xmm11
 2938        pslldq	xmm10, 8
 2939        psrldq	xmm9, 8
 2940        pxor	xmm8, xmm10
 2941        pxor	xmm7, xmm9
 2942        movdqa	xmm12, xmm8
 2943        movdqa	xmm13, xmm8
 2944        movdqa	xmm14, xmm8
 2945        pslld	xmm12, 31
 2946        pslld	xmm13, 30
 2947        pslld	xmm14, 25
 2948        pxor	xmm12, xmm13
 2949        pxor	xmm12, xmm14
 2950        movdqa	xmm13, xmm12
 2951        psrldq	xmm13, 4
 2952        pslldq	xmm12, 12
 2953        pxor	xmm8, xmm12
 2954        movdqa	xmm14, xmm8
 2955        movdqa	xmm10, xmm8
 2956        movdqa	xmm9, xmm8
 2957        psrld	xmm14, 1
 2958        psrld	xmm10, 2
 2959        psrld	xmm9, 7
 2960        pxor	xmm14, xmm10
 2961        pxor	xmm14, xmm9
 2962        pxor	xmm14, xmm13
 2963        pxor	xmm14, xmm8
 2964        pxor	xmm7, xmm14
 2965        movdqu	OWORD PTR [rsp+80], xmm7
 2966        ; H ^ 7
 2967        pshufd	xmm9, xmm1, 78
 2968        pshufd	xmm10, xmm3, 78
 2969        movdqa	xmm11, xmm3
 2970        movdqa	xmm8, xmm3
 2971        pclmulqdq	xmm11, xmm1, 17
 2972        pclmulqdq	xmm8, xmm1, 0
 2973        pxor	xmm9, xmm1
 2974        pxor	xmm10, xmm3
 2975        pclmulqdq	xmm9, xmm10, 0
 2976        pxor	xmm9, xmm8
 2977        pxor	xmm9, xmm11
 2978        movdqa	xmm10, xmm9
 2979        movdqa	xmm7, xmm11
 2980        pslldq	xmm10, 8
 2981        psrldq	xmm9, 8
 2982        pxor	xmm8, xmm10
 2983        pxor	xmm7, xmm9
 2984        movdqa	xmm12, xmm8
 2985        movdqa	xmm13, xmm8
 2986        movdqa	xmm14, xmm8
 2987        pslld	xmm12, 31
 2988        pslld	xmm13, 30
 2989        pslld	xmm14, 25
 2990        pxor	xmm12, xmm13
 2991        pxor	xmm12, xmm14
 2992        movdqa	xmm13, xmm12
 2993        psrldq	xmm13, 4
 2994        pslldq	xmm12, 12
 2995        pxor	xmm8, xmm12
 2996        movdqa	xmm14, xmm8
 2997        movdqa	xmm10, xmm8
 2998        movdqa	xmm9, xmm8
 2999        psrld	xmm14, 1
 3000        psrld	xmm10, 2
 3001        psrld	xmm9, 7
 3002        pxor	xmm14, xmm10
 3003        pxor	xmm14, xmm9
 3004        pxor	xmm14, xmm13
 3005        pxor	xmm14, xmm8
 3006        pxor	xmm7, xmm14
 3007        movdqu	OWORD PTR [rsp+96], xmm7
 3008        ; H ^ 8
 3009        pshufd	xmm9, xmm3, 78
 3010        pshufd	xmm10, xmm3, 78
 3011        movdqa	xmm11, xmm3
 3012        movdqa	xmm8, xmm3
 3013        pclmulqdq	xmm11, xmm3, 17
 3014        pclmulqdq	xmm8, xmm3, 0
 3015        pxor	xmm9, xmm3
 3016        pxor	xmm10, xmm3
 3017        pclmulqdq	xmm9, xmm10, 0
 3018        pxor	xmm9, xmm8
 3019        pxor	xmm9, xmm11
 3020        movdqa	xmm10, xmm9
 3021        movdqa	xmm7, xmm11
 3022        pslldq	xmm10, 8
 3023        psrldq	xmm9, 8
 3024        pxor	xmm8, xmm10
 3025        pxor	xmm7, xmm9
 3026        movdqa	xmm12, xmm8
 3027        movdqa	xmm13, xmm8
 3028        movdqa	xmm14, xmm8
 3029        pslld	xmm12, 31
 3030        pslld	xmm13, 30
 3031        pslld	xmm14, 25
 3032        pxor	xmm12, xmm13
 3033        pxor	xmm12, xmm14
 3034        movdqa	xmm13, xmm12
 3035        psrldq	xmm13, 4
 3036        pslldq	xmm12, 12
 3037        pxor	xmm8, xmm12
 3038        movdqa	xmm14, xmm8
 3039        movdqa	xmm10, xmm8
 3040        movdqa	xmm9, xmm8
 3041        psrld	xmm14, 1
 3042        psrld	xmm10, 2
 3043        psrld	xmm9, 7
 3044        pxor	xmm14, xmm10
 3045        pxor	xmm14, xmm9
 3046        pxor	xmm14, xmm13
 3047        pxor	xmm14, xmm8
 3048        pxor	xmm7, xmm14
 3049        movdqu	OWORD PTR [rsp+112], xmm7
 3050L_AES_GCM_decrypt_aesni_ghash_128:
 3051        lea	rcx, QWORD PTR [rdi+rbx]
 3052        lea	rdx, QWORD PTR [rsi+rbx]
 3053        movdqu	xmm8, OWORD PTR [rsp+128]
 3054        movdqa	xmm1, OWORD PTR L_aes_gcm_bswap_epi64
 3055        movdqa	xmm0, xmm8
 3056        pshufb	xmm8, xmm1
 3057        movdqa	xmm9, xmm0
 3058        paddd	xmm9, OWORD PTR L_aes_gcm_one
 3059        pshufb	xmm9, xmm1
 3060        movdqa	xmm10, xmm0
 3061        paddd	xmm10, OWORD PTR L_aes_gcm_two
 3062        pshufb	xmm10, xmm1
 3063        movdqa	xmm11, xmm0
 3064        paddd	xmm11, OWORD PTR L_aes_gcm_three
 3065        pshufb	xmm11, xmm1
 3066        movdqa	xmm12, xmm0
 3067        paddd	xmm12, OWORD PTR L_aes_gcm_four
 3068        pshufb	xmm12, xmm1
 3069        movdqa	xmm13, xmm0
 3070        paddd	xmm13, OWORD PTR L_aes_gcm_five
 3071        pshufb	xmm13, xmm1
 3072        movdqa	xmm14, xmm0
 3073        paddd	xmm14, OWORD PTR L_aes_gcm_six
 3074        pshufb	xmm14, xmm1
 3075        movdqa	xmm15, xmm0
 3076        paddd	xmm15, OWORD PTR L_aes_gcm_seven
 3077        pshufb	xmm15, xmm1
 3078        paddd	xmm0, OWORD PTR L_aes_gcm_eight
 3079        movdqa	xmm7, OWORD PTR [r15]
 3080        movdqu	OWORD PTR [rsp+128], xmm0
 3081        pxor	xmm8, xmm7
 3082        pxor	xmm9, xmm7
 3083        pxor	xmm10, xmm7
 3084        pxor	xmm11, xmm7
 3085        pxor	xmm12, xmm7
 3086        pxor	xmm13, xmm7
 3087        pxor	xmm14, xmm7
 3088        pxor	xmm15, xmm7
 3089        movdqu	xmm7, OWORD PTR [rsp+112]
 3090        movdqu	xmm0, OWORD PTR [rcx]
 3091        aesenc	xmm8, [r15+16]
 3092        pshufb	xmm0, OWORD PTR L_aes_gcm_bswap_mask
 3093        pxor	xmm0, xmm2
 3094        pshufd	xmm1, xmm7, 78
 3095        pshufd	xmm5, xmm0, 78
 3096        pxor	xmm1, xmm7
 3097        pxor	xmm5, xmm0
 3098        movdqa	xmm3, xmm0
 3099        pclmulqdq	xmm3, xmm7, 17
 3100        aesenc	xmm9, [r15+16]
 3101        aesenc	xmm10, [r15+16]
 3102        movdqa	xmm2, xmm0
 3103        pclmulqdq	xmm2, xmm7, 0
 3104        aesenc	xmm11, [r15+16]
 3105        aesenc	xmm12, [r15+16]
 3106        pclmulqdq	xmm1, xmm5, 0
 3107        aesenc	xmm13, [r15+16]
 3108        aesenc	xmm14, [r15+16]
 3109        aesenc	xmm15, [r15+16]
 3110        pxor	xmm1, xmm2
 3111        pxor	xmm1, xmm3
 3112        movdqu	xmm7, OWORD PTR [rsp+96]
 3113        movdqu	xmm0, OWORD PTR [rcx+16]
 3114        pshufd	xmm4, xmm7, 78
 3115        pshufb	xmm0, OWORD PTR L_aes_gcm_bswap_mask
 3116        aesenc	xmm8, [r15+32]
 3117        pxor	xmm4, xmm7
 3118        pshufd	xmm5, xmm0, 78
 3119        pxor	xmm5, xmm0
 3120        movdqa	xmm6, xmm0
 3121        pclmulqdq	xmm6, xmm7, 17
 3122        aesenc	xmm9, [r15+32]
 3123        aesenc	xmm10, [r15+32]
 3124        pclmulqdq	xmm7, xmm0, 0
 3125        aesenc	xmm11, [r15+32]
 3126        aesenc	xmm12, [r15+32]
 3127        pclmulqdq	xmm4, xmm5, 0
 3128        aesenc	xmm13, [r15+32]
 3129        aesenc	xmm14, [r15+32]
 3130        aesenc	xmm15, [r15+32]
 3131        pxor	xmm1, xmm7
 3132        pxor	xmm2, xmm7
 3133        pxor	xmm1, xmm6
 3134        pxor	xmm3, xmm6
 3135        pxor	xmm1, xmm4
 3136        movdqu	xmm7, OWORD PTR [rsp+80]
 3137        movdqu	xmm0, OWORD PTR [rcx+32]
 3138        pshufd	xmm4, xmm7, 78
 3139        pshufb	xmm0, OWORD PTR L_aes_gcm_bswap_mask
 3140        aesenc	xmm8, [r15+48]
 3141        pxor	xmm4, xmm7
 3142        pshufd	xmm5, xmm0, 78
 3143        pxor	xmm5, xmm0
 3144        movdqa	xmm6, xmm0
 3145        pclmulqdq	xmm6, xmm7, 17
 3146        aesenc	xmm9, [r15+48]
 3147        aesenc	xmm10, [r15+48]
 3148        pclmulqdq	xmm7, xmm0, 0
 3149        aesenc	xmm11, [r15+48]
 3150        aesenc	xmm12, [r15+48]
 3151        pclmulqdq	xmm4, xmm5, 0
 3152        aesenc	xmm13, [r15+48]
 3153        aesenc	xmm14, [r15+48]
 3154        aesenc	xmm15, [r15+48]
 3155        pxor	xmm1, xmm7
 3156        pxor	xmm2, xmm7
 3157        pxor	xmm1, xmm6
 3158        pxor	xmm3, xmm6
 3159        pxor	xmm1, xmm4
 3160        movdqu	xmm7, OWORD PTR [rsp+64]
 3161        movdqu	xmm0, OWORD PTR [rcx+48]
 3162        pshufd	xmm4, xmm7, 78
 3163        pshufb	xmm0, OWORD PTR L_aes_gcm_bswap_mask
 3164        aesenc	xmm8, [r15+64]
 3165        pxor	xmm4, xmm7
 3166        pshufd	xmm5, xmm0, 78
 3167        pxor	xmm5, xmm0
 3168        movdqa	xmm6, xmm0
 3169        pclmulqdq	xmm6, xmm7, 17
 3170        aesenc	xmm9, [r15+64]
 3171        aesenc	xmm10, [r15+64]
 3172        pclmulqdq	xmm7, xmm0, 0
 3173        aesenc	xmm11, [r15+64]
 3174        aesenc	xmm12, [r15+64]
 3175        pclmulqdq	xmm4, xmm5, 0
 3176        aesenc	xmm13, [r15+64]
 3177        aesenc	xmm14, [r15+64]
 3178        aesenc	xmm15, [r15+64]
 3179        pxor	xmm1, xmm7
 3180        pxor	xmm2, xmm7
 3181        pxor	xmm1, xmm6
 3182        pxor	xmm3, xmm6
 3183        pxor	xmm1, xmm4
 3184        movdqu	xmm7, OWORD PTR [rsp+48]
 3185        movdqu	xmm0, OWORD PTR [rcx+64]
 3186        pshufd	xmm4, xmm7, 78
 3187        pshufb	xmm0, OWORD PTR L_aes_gcm_bswap_mask
 3188        aesenc	xmm8, [r15+80]
 3189        pxor	xmm4, xmm7
 3190        pshufd	xmm5, xmm0, 78
 3191        pxor	xmm5, xmm0
 3192        movdqa	xmm6, xmm0
 3193        pclmulqdq	xmm6, xmm7, 17
 3194        aesenc	xmm9, [r15+80]
 3195        aesenc	xmm10, [r15+80]
 3196        pclmulqdq	xmm7, xmm0, 0
 3197        aesenc	xmm11, [r15+80]
 3198        aesenc	xmm12, [r15+80]
 3199        pclmulqdq	xmm4, xmm5, 0
 3200        aesenc	xmm13, [r15+80]
 3201        aesenc	xmm14, [r15+80]
 3202        aesenc	xmm15, [r15+80]
 3203        pxor	xmm1, xmm7
 3204        pxor	xmm2, xmm7
 3205        pxor	xmm1, xmm6
 3206        pxor	xmm3, xmm6
 3207        pxor	xmm1, xmm4
 3208        movdqu	xmm7, OWORD PTR [rsp+32]
 3209        movdqu	xmm0, OWORD PTR [rcx+80]
 3210        pshufd	xmm4, xmm7, 78
 3211        pshufb	xmm0, OWORD PTR L_aes_gcm_bswap_mask
 3212        aesenc	xmm8, [r15+96]
 3213        pxor	xmm4, xmm7
 3214        pshufd	xmm5, xmm0, 78
 3215        pxor	xmm5, xmm0
 3216        movdqa	xmm6, xmm0
 3217        pclmulqdq	xmm6, xmm7, 17
 3218        aesenc	xmm9, [r15+96]
 3219        aesenc	xmm10, [r15+96]
 3220        pclmulqdq	xmm7, xmm0, 0
 3221        aesenc	xmm11, [r15+96]
 3222        aesenc	xmm12, [r15+96]
 3223        pclmulqdq	xmm4, xmm5, 0
 3224        aesenc	xmm13, [r15+96]
 3225        aesenc	xmm14, [r15+96]
 3226        aesenc	xmm15, [r15+96]
 3227        pxor	xmm1, xmm7
 3228        pxor	xmm2, xmm7
 3229        pxor	xmm1, xmm6
 3230        pxor	xmm3, xmm6
 3231        pxor	xmm1, xmm4
 3232        movdqu	xmm7, OWORD PTR [rsp+16]
 3233        movdqu	xmm0, OWORD PTR [rcx+96]
 3234        pshufd	xmm4, xmm7, 78
 3235        pshufb	xmm0, OWORD PTR L_aes_gcm_bswap_mask
 3236        aesenc	xmm8, [r15+112]
 3237        pxor	xmm4, xmm7
 3238        pshufd	xmm5, xmm0, 78
 3239        pxor	xmm5, xmm0
 3240        movdqa	xmm6, xmm0
 3241        pclmulqdq	xmm6, xmm7, 17
 3242        aesenc	xmm9, [r15+112]
 3243        aesenc	xmm10, [r15+112]
 3244        pclmulqdq	xmm7, xmm0, 0
 3245        aesenc	xmm11, [r15+112]
 3246        aesenc	xmm12, [r15+112]
 3247        pclmulqdq	xmm4, xmm5, 0
 3248        aesenc	xmm13, [r15+112]
 3249        aesenc	xmm14, [r15+112]
 3250        aesenc	xmm15, [r15+112]
 3251        pxor	xmm1, xmm7
 3252        pxor	xmm2, xmm7
 3253        pxor	xmm1, xmm6
 3254        pxor	xmm3, xmm6
 3255        pxor	xmm1, xmm4
 3256        movdqu	xmm7, OWORD PTR [rsp]
 3257        movdqu	xmm0, OWORD PTR [rcx+112]
 3258        pshufd	xmm4, xmm7, 78
 3259        pshufb	xmm0, OWORD PTR L_aes_gcm_bswap_mask
 3260        aesenc	xmm8, [r15+128]
 3261        pxor	xmm4, xmm7
 3262        pshufd	xmm5, xmm0, 78
 3263        pxor	xmm5, xmm0
 3264        movdqa	xmm6, xmm0
 3265        pclmulqdq	xmm6, xmm7, 17
 3266        aesenc	xmm9, [r15+128]
 3267        aesenc	xmm10, [r15+128]
 3268        pclmulqdq	xmm7, xmm0, 0
 3269        aesenc	xmm11, [r15+128]
 3270        aesenc	xmm12, [r15+128]
 3271        pclmulqdq	xmm4, xmm5, 0
 3272        aesenc	xmm13, [r15+128]
 3273        aesenc	xmm14, [r15+128]
 3274        aesenc	xmm15, [r15+128]
 3275        pxor	xmm1, xmm7
 3276        pxor	xmm2, xmm7
 3277        pxor	xmm1, xmm6
 3278        pxor	xmm3, xmm6
 3279        pxor	xmm1, xmm4
 3280        movdqa	xmm5, xmm1
 3281        psrldq	xmm1, 8
 3282        pslldq	xmm5, 8
 3283        aesenc	xmm8, [r15+144]
 3284        pxor	xmm2, xmm5
 3285        pxor	xmm3, xmm1
 3286        movdqa	xmm7, xmm2
 3287        movdqa	xmm4, xmm2
 3288        movdqa	xmm5, xmm2
 3289        aesenc	xmm9, [r15+144]
 3290        pslld	xmm7, 31
 3291        pslld	xmm4, 30
 3292        pslld	xmm5, 25
 3293        aesenc	xmm10, [r15+144]
 3294        pxor	xmm7, xmm4
 3295        pxor	xmm7, xmm5
 3296        aesenc	xmm11, [r15+144]
 3297        movdqa	xmm4, xmm7
 3298        pslldq	xmm7, 12
 3299        psrldq	xmm4, 4
 3300        aesenc	xmm12, [r15+144]
 3301        pxor	xmm2, xmm7
 3302        movdqa	xmm5, xmm2
 3303        movdqa	xmm1, xmm2
 3304        movdqa	xmm0, xmm2
 3305        aesenc	xmm13, [r15+144]
 3306        psrld	xmm5, 1
 3307        psrld	xmm1, 2
 3308        psrld	xmm0, 7
 3309        aesenc	xmm14, [r15+144]
 3310        pxor	xmm5, xmm1
 3311        pxor	xmm5, xmm0
 3312        aesenc	xmm15, [r15+144]
 3313        pxor	xmm5, xmm4
 3314        pxor	xmm2, xmm5
 3315        pxor	xmm2, xmm3
 3316        cmp	r10d, 11
 3317        movdqa	xmm7, OWORD PTR [r15+160]
 3318        jl	L_AES_GCM_decrypt_aesni_aesenc_128_ghash_avx_done
 3319        aesenc	xmm8, xmm7
 3320        aesenc	xmm9, xmm7
 3321        aesenc	xmm10, xmm7
 3322        aesenc	xmm11, xmm7
 3323        aesenc	xmm12, xmm7
 3324        aesenc	xmm13, xmm7
 3325        aesenc	xmm14, xmm7
 3326        aesenc	xmm15, xmm7
 3327        movdqa	xmm7, OWORD PTR [r15+176]
 3328        aesenc	xmm8, xmm7
 3329        aesenc	xmm9, xmm7
 3330        aesenc	xmm10, xmm7
 3331        aesenc	xmm11, xmm7
 3332        aesenc	xmm12, xmm7
 3333        aesenc	xmm13, xmm7
 3334        aesenc	xmm14, xmm7
 3335        aesenc	xmm15, xmm7
 3336        cmp	r10d, 13
 3337        movdqa	xmm7, OWORD PTR [r15+192]
 3338        jl	L_AES_GCM_decrypt_aesni_aesenc_128_ghash_avx_done
 3339        aesenc	xmm8, xmm7
 3340        aesenc	xmm9, xmm7
 3341        aesenc	xmm10, xmm7
 3342        aesenc	xmm11, xmm7
 3343        aesenc	xmm12, xmm7
 3344        aesenc	xmm13, xmm7
 3345        aesenc	xmm14, xmm7
 3346        aesenc	xmm15, xmm7
 3347        movdqa	xmm7, OWORD PTR [r15+208]
 3348        aesenc	xmm8, xmm7
 3349        aesenc	xmm9, xmm7
 3350        aesenc	xmm10, xmm7
 3351        aesenc	xmm11, xmm7
 3352        aesenc	xmm12, xmm7
 3353        aesenc	xmm13, xmm7
 3354        aesenc	xmm14, xmm7
 3355        aesenc	xmm15, xmm7
 3356        movdqa	xmm7, OWORD PTR [r15+224]
 3357L_AES_GCM_decrypt_aesni_aesenc_128_ghash_avx_done:
 3358        aesenclast	xmm8, xmm7
 3359        aesenclast	xmm9, xmm7
 3360        movdqu	xmm0, OWORD PTR [rcx]
 3361        movdqu	xmm1, OWORD PTR [rcx+16]
 3362        pxor	xmm8, xmm0
 3363        pxor	xmm9, xmm1
 3364        movdqu	OWORD PTR [rdx], xmm8
 3365        movdqu	OWORD PTR [rdx+16], xmm9
 3366        aesenclast	xmm10, xmm7
 3367        aesenclast	xmm11, xmm7
 3368        movdqu	xmm0, OWORD PTR [rcx+32]
 3369        movdqu	xmm1, OWORD PTR [rcx+48]
 3370        pxor	xmm10, xmm0
 3371        pxor	xmm11, xmm1
 3372        movdqu	OWORD PTR [rdx+32], xmm10
 3373        movdqu	OWORD PTR [rdx+48], xmm11
 3374        aesenclast	xmm12, xmm7
 3375        aesenclast	xmm13, xmm7
 3376        movdqu	xmm0, OWORD PTR [rcx+64]
 3377        movdqu	xmm1, OWORD PTR [rcx+80]
 3378        pxor	xmm12, xmm0
 3379        pxor	xmm13, xmm1
 3380        movdqu	OWORD PTR [rdx+64], xmm12
 3381        movdqu	OWORD PTR [rdx+80], xmm13
 3382        aesenclast	xmm14, xmm7
 3383        aesenclast	xmm15, xmm7
 3384        movdqu	xmm0, OWORD PTR [rcx+96]
 3385        movdqu	xmm1, OWORD PTR [rcx+112]
 3386        pxor	xmm14, xmm0
 3387        pxor	xmm15, xmm1
 3388        movdqu	OWORD PTR [rdx+96], xmm14
 3389        movdqu	OWORD PTR [rdx+112], xmm15
 3390        add	ebx, 128
 3391        cmp	ebx, r13d
 3392        jl	L_AES_GCM_decrypt_aesni_ghash_128
 3393        movdqa	xmm6, xmm2
 3394        movdqu	xmm5, OWORD PTR [rsp]
 3395L_AES_GCM_decrypt_aesni_done_128:
 3396        mov	edx, r9d
 3397        cmp	ebx, edx
 3398        jge	L_AES_GCM_decrypt_aesni_done_dec
 3399        mov	r13d, r9d
 3400        and	r13d, 4294967280
 3401        cmp	ebx, r13d
 3402        jge	L_AES_GCM_decrypt_aesni_last_block_done
 3403L_AES_GCM_decrypt_aesni_last_block_start:
 3404        lea	rcx, QWORD PTR [rdi+rbx]
 3405        lea	rdx, QWORD PTR [rsi+rbx]
 3406        movdqu	xmm1, OWORD PTR [rcx]
 3407        movdqa	xmm0, xmm5
 3408        pshufb	xmm1, OWORD PTR L_aes_gcm_bswap_mask
 3409        pxor	xmm1, xmm6
 3410        movdqu	xmm8, OWORD PTR [rsp+128]
 3411        movdqa	xmm9, xmm8
 3412        pshufb	xmm8, OWORD PTR L_aes_gcm_bswap_epi64
 3413        paddd	xmm9, OWORD PTR L_aes_gcm_one
 3414        pxor	xmm8, [r15]
 3415        movdqu	OWORD PTR [rsp+128], xmm9
 3416        movdqa	xmm10, xmm1
 3417        pclmulqdq	xmm10, xmm0, 16
 3418        aesenc	xmm8, [r15+16]
 3419        aesenc	xmm8, [r15+32]
 3420        movdqa	xmm11, xmm1
 3421        pclmulqdq	xmm11, xmm0, 1
 3422        aesenc	xmm8, [r15+48]
 3423        aesenc	xmm8, [r15+64]
 3424        movdqa	xmm12, xmm1
 3425        pclmulqdq	xmm12, xmm0, 0
 3426        aesenc	xmm8, [r15+80]
 3427        movdqa	xmm1, xmm1
 3428        pclmulqdq	xmm1, xmm0, 17
 3429        aesenc	xmm8, [r15+96]
 3430        pxor	xmm10, xmm11
 3431        movdqa	xmm2, xmm10
 3432        psrldq	xmm10, 8
 3433        pslldq	xmm2, 8
 3434        aesenc	xmm8, [r15+112]
 3435        movdqa	xmm3, xmm1
 3436        pxor	xmm2, xmm12
 3437        pxor	xmm3, xmm10
 3438        movdqa	xmm0, OWORD PTR L_aes_gcm_mod2_128
 3439        movdqa	xmm11, xmm2
 3440        pclmulqdq	xmm11, xmm0, 16
 3441        aesenc	xmm8, [r15+128]
 3442        pshufd	xmm10, xmm2, 78
 3443        pxor	xmm10, xmm11
 3444        movdqa	xmm11, xmm10
 3445        pclmulqdq	xmm11, xmm0, 16
 3446        aesenc	xmm8, [r15+144]
 3447        pshufd	xmm6, xmm10, 78
 3448        pxor	xmm6, xmm11
 3449        pxor	xmm6, xmm3
 3450        cmp	r10d, 11
 3451        movdqa	xmm9, OWORD PTR [r15+160]
 3452        jl	L_AES_GCM_decrypt_aesni_aesenc_gfmul_last
 3453        aesenc	xmm8, xmm9
 3454        aesenc	xmm8, [r15+176]
 3455        cmp	r10d, 13
 3456        movdqa	xmm9, OWORD PTR [r15+192]
 3457        jl	L_AES_GCM_decrypt_aesni_aesenc_gfmul_last
 3458        aesenc	xmm8, xmm9
 3459        aesenc	xmm8, [r15+208]
 3460        movdqa	xmm9, OWORD PTR [r15+224]
 3461L_AES_GCM_decrypt_aesni_aesenc_gfmul_last:
 3462        aesenclast	xmm8, xmm9
 3463        movdqu	xmm9, OWORD PTR [rcx]
 3464        pxor	xmm8, xmm9
 3465        movdqu	OWORD PTR [rdx], xmm8
 3466        add	ebx, 16
 3467        cmp	ebx, r13d
 3468        jl	L_AES_GCM_decrypt_aesni_last_block_start
 3469L_AES_GCM_decrypt_aesni_last_block_done:
 3470        mov	ecx, r9d
 3471        mov	edx, ecx
 3472        and	ecx, 15
 3473        jz	L_AES_GCM_decrypt_aesni_aesenc_last15_dec_avx_done
 3474        movdqu	xmm4, OWORD PTR [rsp+128]
 3475        pshufb	xmm4, OWORD PTR L_aes_gcm_bswap_epi64
 3476        pxor	xmm4, [r15]
 3477        aesenc	xmm4, [r15+16]
 3478        aesenc	xmm4, [r15+32]
 3479        aesenc	xmm4, [r15+48]
 3480        aesenc	xmm4, [r15+64]
 3481        aesenc	xmm4, [r15+80]
 3482        aesenc	xmm4, [r15+96]
 3483        aesenc	xmm4, [r15+112]
 3484        aesenc	xmm4, [r15+128]
 3485        aesenc	xmm4, [r15+144]
 3486        cmp	r10d, 11
 3487        movdqa	xmm9, OWORD PTR [r15+160]
 3488        jl	L_AES_GCM_decrypt_aesni_aesenc_last15_dec_avx_aesenc_avx_last
 3489        aesenc	xmm4, xmm9
 3490        aesenc	xmm4, [r15+176]
 3491        cmp	r10d, 13
 3492        movdqa	xmm9, OWORD PTR [r15+192]
 3493        jl	L_AES_GCM_decrypt_aesni_aesenc_last15_dec_avx_aesenc_avx_last
 3494        aesenc	xmm4, xmm9
 3495        aesenc	xmm4, [r15+208]
 3496        movdqa	xmm9, OWORD PTR [r15+224]
 3497L_AES_GCM_decrypt_aesni_aesenc_last15_dec_avx_aesenc_avx_last:
 3498        aesenclast	xmm4, xmm9
 3499        sub	rsp, 32
 3500        xor	ecx, ecx
 3501        movdqu	OWORD PTR [rsp], xmm4
 3502        pxor	xmm0, xmm0
 3503        movdqu	OWORD PTR [rsp+16], xmm0
 3504L_AES_GCM_decrypt_aesni_aesenc_last15_dec_avx_loop:
 3505        movzx	r13d, BYTE PTR [rdi+rbx]
 3506        mov	BYTE PTR [rsp+rcx+16], r13b
 3507        xor	r13b, BYTE PTR [rsp+rcx]
 3508        mov	BYTE PTR [rsi+rbx], r13b
 3509        inc	ebx
 3510        inc	ecx
 3511        cmp	ebx, edx
 3512        jl	L_AES_GCM_decrypt_aesni_aesenc_last15_dec_avx_loop
 3513        movdqu	xmm4, OWORD PTR [rsp+16]
 3514        add	rsp, 32
 3515        pshufb	xmm4, OWORD PTR L_aes_gcm_bswap_mask
 3516        pxor	xmm6, xmm4
 3517        pshufd	xmm9, xmm5, 78
 3518        pshufd	xmm10, xmm6, 78
 3519        movdqa	xmm11, xmm6
 3520        movdqa	xmm8, xmm6
 3521        pclmulqdq	xmm11, xmm5, 17
 3522        pclmulqdq	xmm8, xmm5, 0
 3523        pxor	xmm9, xmm5
 3524        pxor	xmm10, xmm6
 3525        pclmulqdq	xmm9, xmm10, 0
 3526        pxor	xmm9, xmm8
 3527        pxor	xmm9, xmm11
 3528        movdqa	xmm10, xmm9
 3529        movdqa	xmm6, xmm11
 3530        pslldq	xmm10, 8
 3531        psrldq	xmm9, 8
 3532        pxor	xmm8, xmm10
 3533        pxor	xmm6, xmm9
 3534        movdqa	xmm12, xmm8
 3535        movdqa	xmm13, xmm8
 3536        movdqa	xmm14, xmm8
 3537        pslld	xmm12, 31
 3538        pslld	xmm13, 30
 3539        pslld	xmm14, 25
 3540        pxor	xmm12, xmm13
 3541        pxor	xmm12, xmm14
 3542        movdqa	xmm13, xmm12
 3543        psrldq	xmm13, 4
 3544        pslldq	xmm12, 12
 3545        pxor	xmm8, xmm12
 3546        movdqa	xmm14, xmm8
 3547        movdqa	xmm10, xmm8
 3548        movdqa	xmm9, xmm8
 3549        psrld	xmm14, 1
 3550        psrld	xmm10, 2
 3551        psrld	xmm9, 7
 3552        pxor	xmm14, xmm10
 3553        pxor	xmm14, xmm9
 3554        pxor	xmm14, xmm13
 3555        pxor	xmm14, xmm8
 3556        pxor	xmm6, xmm14
 3557L_AES_GCM_decrypt_aesni_aesenc_last15_dec_avx_done:
 3558L_AES_GCM_decrypt_aesni_done_dec:
 3559        mov	edx, r9d
 3560        mov	ecx, r11d
 3561        shl	rdx, 3
 3562        shl	rcx, 3
 3563        pinsrq	xmm0, rdx, 0
 3564        pinsrq	xmm0, rcx, 1
 3565        pxor	xmm6, xmm0
 3566        pshufd	xmm9, xmm5, 78
 3567        pshufd	xmm10, xmm6, 78
 3568        movdqa	xmm11, xmm6
 3569        movdqa	xmm8, xmm6
 3570        pclmulqdq	xmm11, xmm5, 17
 3571        pclmulqdq	xmm8, xmm5, 0
 3572        pxor	xmm9, xmm5
 3573        pxor	xmm10, xmm6
 3574        pclmulqdq	xmm9, xmm10, 0
 3575        pxor	xmm9, xmm8
 3576        pxor	xmm9, xmm11
 3577        movdqa	xmm10, xmm9
 3578        movdqa	xmm6, xmm11
 3579        pslldq	xmm10, 8
 3580        psrldq	xmm9, 8
 3581        pxor	xmm8, xmm10
 3582        pxor	xmm6, xmm9
 3583        movdqa	xmm12, xmm8
 3584        movdqa	xmm13, xmm8
 3585        movdqa	xmm14, xmm8
 3586        pslld	xmm12, 31
 3587        pslld	xmm13, 30
 3588        pslld	xmm14, 25
 3589        pxor	xmm12, xmm13
 3590        pxor	xmm12, xmm14
 3591        movdqa	xmm13, xmm12
 3592        psrldq	xmm13, 4
 3593        pslldq	xmm12, 12
 3594        pxor	xmm8, xmm12
 3595        movdqa	xmm14, xmm8
 3596        movdqa	xmm10, xmm8
 3597        movdqa	xmm9, xmm8
 3598        psrld	xmm14, 1
 3599        psrld	xmm10, 2
 3600        psrld	xmm9, 7
 3601        pxor	xmm14, xmm10
 3602        pxor	xmm14, xmm9
 3603        pxor	xmm14, xmm13
 3604        pxor	xmm14, xmm8
 3605        pxor	xmm6, xmm14
 3606        pshufb	xmm6, OWORD PTR L_aes_gcm_bswap_mask
 3607        movdqu	xmm0, OWORD PTR [rsp+144]
 3608        pxor	xmm0, xmm6
 3609        cmp	r14d, 16
 3610        je	L_AES_GCM_decrypt_aesni_cmp_tag_16
 3611        sub	rsp, 16
 3612        xor	rcx, rcx
 3613        xor	rbx, rbx
 3614        movdqu	OWORD PTR [rsp], xmm0
 3615L_AES_GCM_decrypt_aesni_cmp_tag_loop:
 3616        movzx	r13d, BYTE PTR [rsp+rcx]
 3617        xor	r13b, BYTE PTR [r8+rcx]
 3618        or	bl, r13b
 3619        inc	ecx
 3620        cmp	ecx, r14d
 3621        jne	L_AES_GCM_decrypt_aesni_cmp_tag_loop
 3622        cmp	bl, 0
 3623        sete	bl
 3624        add	rsp, 16
 3625        xor	rcx, rcx
 3626        jmp	L_AES_GCM_decrypt_aesni_cmp_tag_done
 3627L_AES_GCM_decrypt_aesni_cmp_tag_16:
 3628        movdqu	xmm1, OWORD PTR [r8]
 3629        pcmpeqb	xmm0, xmm1
 3630        pmovmskb	rdx, xmm0
 3631        ; %%edx == 0xFFFF then return 1 else => return 0
 3632        xor	ebx, ebx
 3633        cmp	edx, 65535
 3634        sete	bl
 3635L_AES_GCM_decrypt_aesni_cmp_tag_done:
 3636        mov	DWORD PTR [rbp], ebx
 3637        movdqu	xmm6, OWORD PTR [rsp+168]
 3638        movdqu	xmm7, OWORD PTR [rsp+184]
 3639        movdqu	xmm8, OWORD PTR [rsp+200]
 3640        movdqu	xmm9, OWORD PTR [rsp+216]
 3641        movdqu	xmm10, OWORD PTR [rsp+232]
 3642        movdqu	xmm11, OWORD PTR [rsp+248]
 3643        movdqu	xmm12, OWORD PTR [rsp+264]
 3644        movdqu	xmm13, OWORD PTR [rsp+280]
 3645        movdqu	xmm14, OWORD PTR [rsp+296]
 3646        movdqu	xmm15, OWORD PTR [rsp+312]
 3647        add	rsp, 328
 3648        pop	rbp
 3649        pop	r15
 3650        pop	r14
 3651        pop	rbx
 3652        pop	r12
 3653        pop	rsi
 3654        pop	rdi
 3655        pop	r13
 3656        ret
 3657AES_GCM_decrypt_aesni ENDP
 3658_TEXT ENDS
 3659_TEXT SEGMENT READONLY PARA
 3660AES_GCM_init_aesni PROC
 3661        push	rdi
 3662        push	rsi
 3663        push	r12
 3664        push	r13
 3665        push	r14
 3666        mov	rdi, rcx
 3667        mov	rsi, rdx
 3668        mov	r10, r8
 3669        mov	r11d, r9d
 3670        mov	rax, QWORD PTR [rsp+80]
 3671        mov	r8, QWORD PTR [rsp+88]
 3672        mov	r9, QWORD PTR [rsp+96]
 3673        sub	rsp, 80
 3674        movdqu	OWORD PTR [rsp+16], xmm6
 3675        movdqu	OWORD PTR [rsp+32], xmm7
 3676        movdqu	OWORD PTR [rsp+48], xmm8
 3677        movdqu	OWORD PTR [rsp+64], xmm15
 3678        pxor	xmm4, xmm4
 3679        mov	edx, r11d
 3680        cmp	edx, 12
 3681        jne	L_AES_GCM_init_aesni_iv_not_12
 3682        ; # Calculate values when IV is 12 bytes
 3683        ; Set counter based on IV
 3684        mov	ecx, 16777216
 3685        pinsrq	xmm4, QWORD PTR [r10], 0
 3686        pinsrd	xmm4, DWORD PTR [r10+8], 2
 3687        pinsrd	xmm4, ecx, 3
 3688        ; H = Encrypt X(=0) and T = Encrypt counter
 3689        movdqa	xmm1, xmm4
 3690        movdqa	xmm5, OWORD PTR [rdi]
 3691        pxor	xmm1, xmm5
 3692        movdqa	xmm6, OWORD PTR [rdi+16]
 3693        aesenc	xmm5, xmm6
 3694        aesenc	xmm1, xmm6
 3695        movdqa	xmm6, OWORD PTR [rdi+32]
 3696        aesenc	xmm5, xmm6
 3697        aesenc	xmm1, xmm6
 3698        movdqa	xmm6, OWORD PTR [rdi+48]
 3699        aesenc	xmm5, xmm6
 3700        aesenc	xmm1, xmm6
 3701        movdqa	xmm6, OWORD PTR [rdi+64]
 3702        aesenc	xmm5, xmm6
 3703        aesenc	xmm1, xmm6
 3704        movdqa	xmm6, OWORD PTR [rdi+80]
 3705        aesenc	xmm5, xmm6
 3706        aesenc	xmm1, xmm6
 3707        movdqa	xmm6, OWORD PTR [rdi+96]
 3708        aesenc	xmm5, xmm6
 3709        aesenc	xmm1, xmm6
 3710        movdqa	xmm6, OWORD PTR [rdi+112]
 3711        aesenc	xmm5, xmm6
 3712        aesenc	xmm1, xmm6
 3713        movdqa	xmm6, OWORD PTR [rdi+128]
 3714        aesenc	xmm5, xmm6
 3715        aesenc	xmm1, xmm6
 3716        movdqa	xmm6, OWORD PTR [rdi+144]
 3717        aesenc	xmm5, xmm6
 3718        aesenc	xmm1, xmm6
 3719        cmp	esi, 11
 3720        movdqa	xmm6, OWORD PTR [rdi+160]
 3721        jl	L_AES_GCM_init_aesni_calc_iv_12_last
 3722        aesenc	xmm5, xmm6
 3723        aesenc	xmm1, xmm6
 3724        movdqa	xmm6, OWORD PTR [rdi+176]
 3725        aesenc	xmm5, xmm6
 3726        aesenc	xmm1, xmm6
 3727        cmp	esi, 13
 3728        movdqa	xmm6, OWORD PTR [rdi+192]
 3729        jl	L_AES_GCM_init_aesni_calc_iv_12_last
 3730        aesenc	xmm5, xmm6
 3731        aesenc	xmm1, xmm6
 3732        movdqa	xmm6, OWORD PTR [rdi+208]
 3733        aesenc	xmm5, xmm6
 3734        aesenc	xmm1, xmm6
 3735        movdqa	xmm6, OWORD PTR [rdi+224]
 3736L_AES_GCM_init_aesni_calc_iv_12_last:
 3737        aesenclast	xmm5, xmm6
 3738        aesenclast	xmm1, xmm6
 3739        pshufb	xmm5, OWORD PTR L_aes_gcm_bswap_mask
 3740        movdqu	xmm15, xmm1
 3741        jmp	L_AES_GCM_init_aesni_iv_done
 3742L_AES_GCM_init_aesni_iv_not_12:
 3743        ; Calculate values when IV is not 12 bytes
 3744        ; H = Encrypt X(=0)
 3745        movdqa	xmm5, OWORD PTR [rdi]
 3746        aesenc	xmm5, [rdi+16]
 3747        aesenc	xmm5, [rdi+32]
 3748        aesenc	xmm5, [rdi+48]
 3749        aesenc	xmm5, [rdi+64]
 3750        aesenc	xmm5, [rdi+80]
 3751        aesenc	xmm5, [rdi+96]
 3752        aesenc	xmm5, [rdi+112]
 3753        aesenc	xmm5, [rdi+128]
 3754        aesenc	xmm5, [rdi+144]
 3755        cmp	esi, 11
 3756        movdqa	xmm8, OWORD PTR [rdi+160]
 3757        jl	L_AES_GCM_init_aesni_calc_iv_1_aesenc_avx_last
 3758        aesenc	xmm5, xmm8
 3759        aesenc	xmm5, [rdi+176]
 3760        cmp	esi, 13
 3761        movdqa	xmm8, OWORD PTR [rdi+192]
 3762        jl	L_AES_GCM_init_aesni_calc_iv_1_aesenc_avx_last
 3763        aesenc	xmm5, xmm8
 3764        aesenc	xmm5, [rdi+208]
 3765        movdqa	xmm8, OWORD PTR [rdi+224]
 3766L_AES_GCM_init_aesni_calc_iv_1_aesenc_avx_last:
 3767        aesenclast	xmm5, xmm8
 3768        pshufb	xmm5, OWORD PTR L_aes_gcm_bswap_mask
 3769        ; Calc counter
 3770        ; Initialization vector
 3771        cmp	edx, 0
 3772        mov	rcx, 0
 3773        je	L_AES_GCM_init_aesni_calc_iv_done
 3774        cmp	edx, 16
 3775        jl	L_AES_GCM_init_aesni_calc_iv_lt16
 3776        and	edx, 4294967280
 3777L_AES_GCM_init_aesni_calc_iv_16_loop:
 3778        movdqu	xmm7, OWORD PTR [r10+rcx]
 3779        pshufb	xmm7, OWORD PTR L_aes_gcm_bswap_mask
 3780        pxor	xmm4, xmm7
 3781        pshufd	xmm1, xmm4, 78
 3782        pshufd	xmm2, xmm5, 78
 3783        movdqa	xmm3, xmm5
 3784        movdqa	xmm0, xmm5
 3785        pclmulqdq	xmm3, xmm4, 17
 3786        pclmulqdq	xmm0, xmm4, 0
 3787        pxor	xmm1, xmm4
 3788        pxor	xmm2, xmm5
 3789        pclmulqdq	xmm1, xmm2, 0
 3790        pxor	xmm1, xmm0
 3791        pxor	xmm1, xmm3
 3792        movdqa	xmm2, xmm1
 3793        movdqa	xmm6, xmm0
 3794        movdqa	xmm4, xmm3
 3795        pslldq	xmm2, 8
 3796        psrldq	xmm1, 8
 3797        pxor	xmm6, xmm2
 3798        pxor	xmm4, xmm1
 3799        movdqa	xmm0, xmm6
 3800        movdqa	xmm1, xmm4
 3801        psrld	xmm0, 31
 3802        psrld	xmm1, 31
 3803        pslld	xmm6, 1
 3804        pslld	xmm4, 1
 3805        movdqa	xmm2, xmm0
 3806        pslldq	xmm0, 4
 3807        psrldq	xmm2, 12
 3808        pslldq	xmm1, 4
 3809        por	xmm4, xmm2
 3810        por	xmm6, xmm0
 3811        por	xmm4, xmm1
 3812        movdqa	xmm0, xmm6
 3813        movdqa	xmm1, xmm6
 3814        movdqa	xmm2, xmm6
 3815        pslld	xmm0, 31
 3816        pslld	xmm1, 30
 3817        pslld	xmm2, 25
 3818        pxor	xmm0, xmm1
 3819        pxor	xmm0, xmm2
 3820        movdqa	xmm1, xmm0
 3821        psrldq	xmm1, 4
 3822        pslldq	xmm0, 12
 3823        pxor	xmm6, xmm0
 3824        movdqa	xmm2, xmm6
 3825        movdqa	xmm3, xmm6
 3826        movdqa	xmm0, xmm6
 3827        psrld	xmm2, 1
 3828        psrld	xmm3, 2
 3829        psrld	xmm0, 7
 3830        pxor	xmm2, xmm3
 3831        pxor	xmm2, xmm0
 3832        pxor	xmm2, xmm1
 3833        pxor	xmm2, xmm6
 3834        pxor	xmm4, xmm2
 3835        add	ecx, 16
 3836        cmp	ecx, edx
 3837        jl	L_AES_GCM_init_aesni_calc_iv_16_loop
 3838        mov	edx, r11d
 3839        cmp	ecx, edx
 3840        je	L_AES_GCM_init_aesni_calc_iv_done
 3841L_AES_GCM_init_aesni_calc_iv_lt16:
 3842        sub	rsp, 16
 3843        pxor	xmm7, xmm7
 3844        xor	r13d, r13d
 3845        movdqu	OWORD PTR [rsp], xmm7
 3846L_AES_GCM_init_aesni_calc_iv_loop:
 3847        movzx	r12d, BYTE PTR [r10+rcx]
 3848        mov	BYTE PTR [rsp+r13], r12b
 3849        inc	ecx
 3850        inc	r13d
 3851        cmp	ecx, edx
 3852        jl	L_AES_GCM_init_aesni_calc_iv_loop
 3853        movdqu	xmm7, OWORD PTR [rsp]
 3854        add	rsp, 16
 3855        pshufb	xmm7, OWORD PTR L_aes_gcm_bswap_mask
 3856        pxor	xmm4, xmm7
 3857        pshufd	xmm1, xmm4, 78
 3858        pshufd	xmm2, xmm5, 78
 3859        movdqa	xmm3, xmm5
 3860        movdqa	xmm0, xmm5
 3861        pclmulqdq	xmm3, xmm4, 17
 3862        pclmulqdq	xmm0, xmm4, 0
 3863        pxor	xmm1, xmm4
 3864        pxor	xmm2, xmm5
 3865        pclmulqdq	xmm1, xmm2, 0
 3866        pxor	xmm1, xmm0
 3867        pxor	xmm1, xmm3
 3868        movdqa	xmm2, xmm1
 3869        movdqa	xmm6, xmm0
 3870        movdqa	xmm4, xmm3
 3871        pslldq	xmm2, 8
 3872        psrldq	xmm1, 8
 3873        pxor	xmm6, xmm2
 3874        pxor	xmm4, xmm1
 3875        movdqa	xmm0, xmm6
 3876        movdqa	xmm1, xmm4
 3877        psrld	xmm0, 31
 3878        psrld	xmm1, 31
 3879        pslld	xmm6, 1
 3880        pslld	xmm4, 1
 3881        movdqa	xmm2, xmm0
 3882        pslldq	xmm0, 4
 3883        psrldq	xmm2, 12
 3884        pslldq	xmm1, 4
 3885        por	xmm4, xmm2
 3886        por	xmm6, xmm0
 3887        por	xmm4, xmm1
 3888        movdqa	xmm0, xmm6
 3889        movdqa	xmm1, xmm6
 3890        movdqa	xmm2, xmm6
 3891        pslld	xmm0, 31
 3892        pslld	xmm1, 30
 3893        pslld	xmm2, 25
 3894        pxor	xmm0, xmm1
 3895        pxor	xmm0, xmm2
 3896        movdqa	xmm1, xmm0
 3897        psrldq	xmm1, 4
 3898        pslldq	xmm0, 12
 3899        pxor	xmm6, xmm0
 3900        movdqa	xmm2, xmm6
 3901        movdqa	xmm3, xmm6
 3902        movdqa	xmm0, xmm6
 3903        psrld	xmm2, 1
 3904        psrld	xmm3, 2
 3905        psrld	xmm0, 7
 3906        pxor	xmm2, xmm3
 3907        pxor	xmm2, xmm0
 3908        pxor	xmm2, xmm1
 3909        pxor	xmm2, xmm6
 3910        pxor	xmm4, xmm2
 3911L_AES_GCM_init_aesni_calc_iv_done:
 3912        ; T = Encrypt counter
 3913        pxor	xmm0, xmm0
 3914        shl	edx, 3
 3915        pinsrq	xmm0, rdx, 0
 3916        pxor	xmm4, xmm0
 3917        pshufd	xmm1, xmm4, 78
 3918        pshufd	xmm2, xmm5, 78
 3919        movdqa	xmm3, xmm5
 3920        movdqa	xmm0, xmm5
 3921        pclmulqdq	xmm3, xmm4, 17
 3922        pclmulqdq	xmm0, xmm4, 0
 3923        pxor	xmm1, xmm4
 3924        pxor	xmm2, xmm5
 3925        pclmulqdq	xmm1, xmm2, 0
 3926        pxor	xmm1, xmm0
 3927        pxor	xmm1, xmm3
 3928        movdqa	xmm2, xmm1
 3929        movdqa	xmm6, xmm0
 3930        movdqa	xmm4, xmm3
 3931        pslldq	xmm2, 8
 3932        psrldq	xmm1, 8
 3933        pxor	xmm6, xmm2
 3934        pxor	xmm4, xmm1
 3935        movdqa	xmm0, xmm6
 3936        movdqa	xmm1, xmm4
 3937        psrld	xmm0, 31
 3938        psrld	xmm1, 31
 3939        pslld	xmm6, 1
 3940        pslld	xmm4, 1
 3941        movdqa	xmm2, xmm0
 3942        pslldq	xmm0, 4
 3943        psrldq	xmm2, 12
 3944        pslldq	xmm1, 4
 3945        por	xmm4, xmm2
 3946        por	xmm6, xmm0
 3947        por	xmm4, xmm1
 3948        movdqa	xmm0, xmm6
 3949        movdqa	xmm1, xmm6
 3950        movdqa	xmm2, xmm6
 3951        pslld	xmm0, 31
 3952        pslld	xmm1, 30
 3953        pslld	xmm2, 25
 3954        pxor	xmm0, xmm1
 3955        pxor	xmm0, xmm2
 3956        movdqa	xmm1, xmm0
 3957        psrldq	xmm1, 4
 3958        pslldq	xmm0, 12
 3959        pxor	xmm6, xmm0
 3960        movdqa	xmm2, xmm6
 3961        movdqa	xmm3, xmm6
 3962        movdqa	xmm0, xmm6
 3963        psrld	xmm2, 1
 3964        psrld	xmm3, 2
 3965        psrld	xmm0, 7
 3966        pxor	xmm2, xmm3
 3967        pxor	xmm2, xmm0
 3968        pxor	xmm2, xmm1
 3969        pxor	xmm2, xmm6
 3970        pxor	xmm4, xmm2
 3971        pshufb	xmm4, OWORD PTR L_aes_gcm_bswap_mask
 3972        ;   Encrypt counter
 3973        movdqa	xmm7, OWORD PTR [rdi]
 3974        pxor	xmm7, xmm4
 3975        aesenc	xmm7, [rdi+16]
 3976        aesenc	xmm7, [rdi+32]
 3977        aesenc	xmm7, [rdi+48]
 3978        aesenc	xmm7, [rdi+64]
 3979        aesenc	xmm7, [rdi+80]
 3980        aesenc	xmm7, [rdi+96]
 3981        aesenc	xmm7, [rdi+112]
 3982        aesenc	xmm7, [rdi+128]
 3983        aesenc	xmm7, [rdi+144]
 3984        cmp	esi, 11
 3985        movdqa	xmm8, OWORD PTR [rdi+160]
 3986        jl	L_AES_GCM_init_aesni_calc_iv_2_aesenc_avx_last
 3987        aesenc	xmm7, xmm8
 3988        aesenc	xmm7, [rdi+176]
 3989        cmp	esi, 13
 3990        movdqa	xmm8, OWORD PTR [rdi+192]
 3991        jl	L_AES_GCM_init_aesni_calc_iv_2_aesenc_avx_last
 3992        aesenc	xmm7, xmm8
 3993        aesenc	xmm7, [rdi+208]
 3994        movdqa	xmm8, OWORD PTR [rdi+224]
 3995L_AES_GCM_init_aesni_calc_iv_2_aesenc_avx_last:
 3996        aesenclast	xmm7, xmm8
 3997        movdqu	xmm15, xmm7
 3998L_AES_GCM_init_aesni_iv_done:
 3999        movdqa	OWORD PTR [r9], xmm15
 4000        pshufb	xmm4, OWORD PTR L_aes_gcm_bswap_epi64
 4001        paddd	xmm4, OWORD PTR L_aes_gcm_one
 4002        movdqa	OWORD PTR [rax], xmm5
 4003        movdqa	OWORD PTR [r8], xmm4
 4004        movdqu	xmm6, OWORD PTR [rsp+16]
 4005        movdqu	xmm7, OWORD PTR [rsp+32]
 4006        movdqu	xmm8, OWORD PTR [rsp+48]
 4007        movdqu	xmm15, OWORD PTR [rsp+64]
 4008        add	rsp, 80
 4009        pop	r14
 4010        pop	r13
 4011        pop	r12
 4012        pop	rsi
 4013        pop	rdi
 4014        ret
 4015AES_GCM_init_aesni ENDP
 4016_TEXT ENDS
 4017_TEXT SEGMENT READONLY PARA
 4018AES_GCM_aad_update_aesni PROC
 4019        mov	rax, rcx
 4020        sub	rsp, 32
 4021        movdqu	OWORD PTR [rsp], xmm6
 4022        movdqu	OWORD PTR [rsp+16], xmm7
 4023        movdqa	xmm5, OWORD PTR [r8]
 4024        movdqa	xmm6, OWORD PTR [r9]
 4025        xor	ecx, ecx
 4026L_AES_GCM_aad_update_aesni_16_loop:
 4027        movdqu	xmm7, OWORD PTR [rax+rcx]
 4028        pshufb	xmm7, OWORD PTR L_aes_gcm_bswap_mask
 4029        pxor	xmm5, xmm7
 4030        pshufd	xmm1, xmm5, 78
 4031        pshufd	xmm2, xmm6, 78
 4032        movdqa	xmm3, xmm6
 4033        movdqa	xmm0, xmm6
 4034        pclmulqdq	xmm3, xmm5, 17
 4035        pclmulqdq	xmm0, xmm5, 0
 4036        pxor	xmm1, xmm5
 4037        pxor	xmm2, xmm6
 4038        pclmulqdq	xmm1, xmm2, 0
 4039        pxor	xmm1, xmm0
 4040        pxor	xmm1, xmm3
 4041        movdqa	xmm2, xmm1
 4042        movdqa	xmm4, xmm0
 4043        movdqa	xmm5, xmm3
 4044        pslldq	xmm2, 8
 4045        psrldq	xmm1, 8
 4046        pxor	xmm4, xmm2
 4047        pxor	xmm5, xmm1
 4048        movdqa	xmm0, xmm4
 4049        movdqa	xmm1, xmm5
 4050        psrld	xmm0, 31
 4051        psrld	xmm1, 31
 4052        pslld	xmm4, 1
 4053        pslld	xmm5, 1
 4054        movdqa	xmm2, xmm0
 4055        pslldq	xmm0, 4
 4056        psrldq	xmm2, 12
 4057        pslldq	xmm1, 4
 4058        por	xmm5, xmm2
 4059        por	xmm4, xmm0
 4060        por	xmm5, xmm1
 4061        movdqa	xmm0, xmm4
 4062        movdqa	xmm1, xmm4
 4063        movdqa	xmm2, xmm4
 4064        pslld	xmm0, 31
 4065        pslld	xmm1, 30
 4066        pslld	xmm2, 25
 4067        pxor	xmm0, xmm1
 4068        pxor	xmm0, xmm2
 4069        movdqa	xmm1, xmm0
 4070        psrldq	xmm1, 4
 4071        pslldq	xmm0, 12
 4072        pxor	xmm4, xmm0
 4073        movdqa	xmm2, xmm4
 4074        movdqa	xmm3, xmm4
 4075        movdqa	xmm0, xmm4
 4076        psrld	xmm2, 1
 4077        psrld	xmm3, 2
 4078        psrld	xmm0, 7
 4079        pxor	xmm2, xmm3
 4080        pxor	xmm2, xmm0
 4081        pxor	xmm2, xmm1
 4082        pxor	xmm2, xmm4
 4083        pxor	xmm5, xmm2
 4084        add	ecx, 16
 4085        cmp	ecx, edx
 4086        jl	L_AES_GCM_aad_update_aesni_16_loop
 4087        movdqa	OWORD PTR [r8], xmm5
 4088        movdqu	xmm6, OWORD PTR [rsp]
 4089        movdqu	xmm7, OWORD PTR [rsp+16]
 4090        add	rsp, 32
 4091        ret
 4092AES_GCM_aad_update_aesni ENDP
 4093_TEXT ENDS
 4094_TEXT SEGMENT READONLY PARA
 4095AES_GCM_encrypt_block_aesni PROC
 4096        mov	r10, r8
 4097        mov	r11, r9
 4098        mov	rax, QWORD PTR [rsp+40]
 4099        movdqu	xmm0, OWORD PTR [rax]
 4100        movdqa	xmm1, xmm0
 4101        pshufb	xmm0, OWORD PTR L_aes_gcm_bswap_epi64
 4102        paddd	xmm1, OWORD PTR L_aes_gcm_one
 4103        pxor	xmm0, [rcx]
 4104        movdqu	OWORD PTR [rax], xmm1
 4105        aesenc	xmm0, [rcx+16]
 4106        aesenc	xmm0, [rcx+32]
 4107        aesenc	xmm0, [rcx+48]
 4108        aesenc	xmm0, [rcx+64]
 4109        aesenc	xmm0, [rcx+80]
 4110        aesenc	xmm0, [rcx+96]
 4111        aesenc	xmm0, [rcx+112]
 4112        aesenc	xmm0, [rcx+128]
 4113        aesenc	xmm0, [rcx+144]
 4114        cmp	edx, 11
 4115        movdqa	xmm1, OWORD PTR [rcx+160]
 4116        jl	L_AES_GCM_encrypt_block_aesni_aesenc_block_aesenc_avx_last
 4117        aesenc	xmm0, xmm1
 4118        aesenc	xmm0, [rcx+176]
 4119        cmp	edx, 13
 4120        movdqa	xmm1, OWORD PTR [rcx+192]
 4121        jl	L_AES_GCM_encrypt_block_aesni_aesenc_block_aesenc_avx_last
 4122        aesenc	xmm0, xmm1
 4123        aesenc	xmm0, [rcx+208]
 4124        movdqa	xmm1, OWORD PTR [rcx+224]
 4125L_AES_GCM_encrypt_block_aesni_aesenc_block_aesenc_avx_last:
 4126        aesenclast	xmm0, xmm1
 4127        movdqu	xmm1, OWORD PTR [r11]
 4128        pxor	xmm0, xmm1
 4129        movdqu	OWORD PTR [r10], xmm0
 4130        pshufb	xmm0, OWORD PTR L_aes_gcm_bswap_mask
 4131        ret
 4132AES_GCM_encrypt_block_aesni ENDP
 4133_TEXT ENDS
 4134_TEXT SEGMENT READONLY PARA
 4135AES_GCM_ghash_block_aesni PROC
 4136        sub	rsp, 32
 4137        movdqu	OWORD PTR [rsp], xmm6
 4138        movdqu	OWORD PTR [rsp+16], xmm7
 4139        movdqa	xmm4, OWORD PTR [rdx]
 4140        movdqa	xmm5, OWORD PTR [r8]
 4141        movdqu	xmm7, OWORD PTR [rcx]
 4142        pshufb	xmm7, OWORD PTR L_aes_gcm_bswap_mask
 4143        pxor	xmm4, xmm7
 4144        pshufd	xmm1, xmm4, 78
 4145        pshufd	xmm2, xmm5, 78
 4146        movdqa	xmm3, xmm5
 4147        movdqa	xmm0, xmm5
 4148        pclmulqdq	xmm3, xmm4, 17
 4149        pclmulqdq	xmm0, xmm4, 0
 4150        pxor	xmm1, xmm4
 4151        pxor	xmm2, xmm5
 4152        pclmulqdq	xmm1, xmm2, 0
 4153        pxor	xmm1, xmm0
 4154        pxor	xmm1, xmm3
 4155        movdqa	xmm2, xmm1
 4156        movdqa	xmm6, xmm0
 4157        movdqa	xmm4, xmm3
 4158        pslldq	xmm2, 8
 4159        psrldq	xmm1, 8
 4160        pxor	xmm6, xmm2
 4161        pxor	xmm4, xmm1
 4162        movdqa	xmm0, xmm6
 4163        movdqa	xmm1, xmm4
 4164        psrld	xmm0, 31
 4165        psrld	xmm1, 31
 4166        pslld	xmm6, 1
 4167        pslld	xmm4, 1
 4168        movdqa	xmm2, xmm0
 4169        pslldq	xmm0, 4
 4170        psrldq	xmm2, 12
 4171        pslldq	xmm1, 4
 4172        por	xmm4, xmm2
 4173        por	xmm6, xmm0
 4174        por	xmm4, xmm1
 4175        movdqa	xmm0, xmm6
 4176        movdqa	xmm1, xmm6
 4177        movdqa	xmm2, xmm6
 4178        pslld	xmm0, 31
 4179        pslld	xmm1, 30
 4180        pslld	xmm2, 25
 4181        pxor	xmm0, xmm1
 4182        pxor	xmm0, xmm2
 4183        movdqa	xmm1, xmm0
 4184        psrldq	xmm1, 4
 4185        pslldq	xmm0, 12
 4186        pxor	xmm6, xmm0
 4187        movdqa	xmm2, xmm6
 4188        movdqa	xmm3, xmm6
 4189        movdqa	xmm0, xmm6
 4190        psrld	xmm2, 1
 4191        psrld	xmm3, 2
 4192        psrld	xmm0, 7
 4193        pxor	xmm2, xmm3
 4194        pxor	xmm2, xmm0
 4195        pxor	xmm2, xmm1
 4196        pxor	xmm2, xmm6
 4197        pxor	xmm4, xmm2
 4198        movdqa	OWORD PTR [rdx], xmm4
 4199        movdqu	xmm6, OWORD PTR [rsp]
 4200        movdqu	xmm7, OWORD PTR [rsp+16]
 4201        add	rsp, 32
 4202        ret
 4203AES_GCM_ghash_block_aesni ENDP
 4204_TEXT ENDS
 4205_TEXT SEGMENT READONLY PARA
 4206AES_GCM_encrypt_update_aesni PROC
 4207        push	r13
 4208        push	r12
 4209        push	r14
 4210        push	r15
 4211        push	rdi
 4212        mov	rax, rcx
 4213        mov	r10, r8
 4214        mov	r8d, edx
 4215        mov	r11, r9
 4216        mov	r9d, DWORD PTR [rsp+80]
 4217        mov	r12, QWORD PTR [rsp+88]
 4218        mov	r14, QWORD PTR [rsp+96]
 4219        mov	r15, QWORD PTR [rsp+104]
 4220        sub	rsp, 320
 4221        movdqu	OWORD PTR [rsp+160], xmm6
 4222        movdqu	OWORD PTR [rsp+176], xmm7
 4223        movdqu	OWORD PTR [rsp+192], xmm8
 4224        movdqu	OWORD PTR [rsp+208], xmm9
 4225        movdqu	OWORD PTR [rsp+224], xmm10
 4226        movdqu	OWORD PTR [rsp+240], xmm11
 4227        movdqu	OWORD PTR [rsp+256], xmm12
 4228        movdqu	OWORD PTR [rsp+272], xmm13
 4229        movdqu	OWORD PTR [rsp+288], xmm14
 4230        movdqu	OWORD PTR [rsp+304], xmm15
 4231        movdqa	xmm6, OWORD PTR [r12]
 4232        movdqa	xmm5, OWORD PTR [r14]
 4233        movdqa	xmm9, xmm5
 4234        movdqa	xmm8, xmm5
 4235        psrlq	xmm9, 63
 4236        psllq	xmm8, 1
 4237        pslldq	xmm9, 8
 4238        por	xmm8, xmm9
 4239        pshufd	xmm5, xmm5, 255
 4240        psrad	xmm5, 31
 4241        pand	xmm5, OWORD PTR L_aes_gcm_mod2_128
 4242        pxor	xmm5, xmm8
 4243        xor	rdi, rdi
 4244        cmp	r9d, 128
 4245        mov	r13d, r9d
 4246        jl	L_AES_GCM_encrypt_update_aesni_done_128
 4247        and	r13d, 4294967168
 4248        movdqa	xmm2, xmm6
 4249        ; H ^ 1
 4250        movdqu	OWORD PTR [rsp], xmm5
 4251        ; H ^ 2
 4252        pshufd	xmm9, xmm5, 78
 4253        pshufd	xmm10, xmm5, 78
 4254        movdqa	xmm11, xmm5
 4255        movdqa	xmm8, xmm5
 4256        pclmulqdq	xmm11, xmm5, 17
 4257        pclmulqdq	xmm8, xmm5, 0
 4258        pxor	xmm9, xmm5
 4259        pxor	xmm10, xmm5
 4260        pclmulqdq	xmm9, xmm10, 0
 4261        pxor	xmm9, xmm8
 4262        pxor	xmm9, xmm11
 4263        movdqa	xmm10, xmm9
 4264        movdqa	xmm0, xmm11
 4265        pslldq	xmm10, 8
 4266        psrldq	xmm9, 8
 4267        pxor	xmm8, xmm10
 4268        pxor	xmm0, xmm9
 4269        movdqa	xmm12, xmm8
 4270        movdqa	xmm13, xmm8
 4271        movdqa	xmm14, xmm8
 4272        pslld	xmm12, 31
 4273        pslld	xmm13, 30
 4274        pslld	xmm14, 25
 4275        pxor	xmm12, xmm13
 4276        pxor	xmm12, xmm14
 4277        movdqa	xmm13, xmm12
 4278        psrldq	xmm13, 4
 4279        pslldq	xmm12, 12
 4280        pxor	xmm8, xmm12
 4281        movdqa	xmm14, xmm8
 4282        movdqa	xmm10, xmm8
 4283        movdqa	xmm9, xmm8
 4284        psrld	xmm14, 1
 4285        psrld	xmm10, 2
 4286        psrld	xmm9, 7
 4287        pxor	xmm14, xmm10
 4288        pxor	xmm14, xmm9
 4289        pxor	xmm14, xmm13
 4290        pxor	xmm14, xmm8
 4291        pxor	xmm0, xmm14
 4292        movdqu	OWORD PTR [rsp+16], xmm0
 4293        ; H ^ 3
 4294        pshufd	xmm9, xmm5, 78
 4295        pshufd	xmm10, xmm0, 78
 4296        movdqa	xmm11, xmm0
 4297        movdqa	xmm8, xmm0
 4298        pclmulqdq	xmm11, xmm5, 17
 4299        pclmulqdq	xmm8, xmm5, 0
 4300        pxor	xmm9, xmm5
 4301        pxor	xmm10, xmm0
 4302        pclmulqdq	xmm9, xmm10, 0
 4303        pxor	xmm9, xmm8
 4304        pxor	xmm9, xmm11
 4305        movdqa	xmm10, xmm9
 4306        movdqa	xmm1, xmm11
 4307        pslldq	xmm10, 8
 4308        psrldq	xmm9, 8
 4309        pxor	xmm8, xmm10
 4310        pxor	xmm1, xmm9
 4311        movdqa	xmm12, xmm8
 4312        movdqa	xmm13, xmm8
 4313        movdqa	xmm14, xmm8
 4314        pslld	xmm12, 31
 4315        pslld	xmm13, 30
 4316        pslld	xmm14, 25
 4317        pxor	xmm12, xmm13
 4318        pxor	xmm12, xmm14
 4319        movdqa	xmm13, xmm12
 4320        psrldq	xmm13, 4
 4321        pslldq	xmm12, 12
 4322        pxor	xmm8, xmm12
 4323        movdqa	xmm14, xmm8
 4324        movdqa	xmm10, xmm8
 4325        movdqa	xmm9, xmm8
 4326        psrld	xmm14, 1
 4327        psrld	xmm10, 2
 4328        psrld	xmm9, 7
 4329        pxor	xmm14, xmm10
 4330        pxor	xmm14, xmm9
 4331        pxor	xmm14, xmm13
 4332        pxor	xmm14, xmm8
 4333        pxor	xmm1, xmm14
 4334        movdqu	OWORD PTR [rsp+32], xmm1
 4335        ; H ^ 4
 4336        pshufd	xmm9, xmm0, 78
 4337        pshufd	xmm10, xmm0, 78
 4338        movdqa	xmm11, xmm0
 4339        movdqa	xmm8, xmm0
 4340        pclmulqdq	xmm11, xmm0, 17
 4341        pclmulqdq	xmm8, xmm0, 0
 4342        pxor	xmm9, xmm0
 4343        pxor	xmm10, xmm0
 4344        pclmulqdq	xmm9, xmm10, 0
 4345        pxor	xmm9, xmm8
 4346        pxor	xmm9, xmm11
 4347        movdqa	xmm10, xmm9
 4348        movdqa	xmm3, xmm11
 4349        pslldq	xmm10, 8
 4350        psrldq	xmm9, 8
 4351        pxor	xmm8, xmm10
 4352        pxor	xmm3, xmm9
 4353        movdqa	xmm12, xmm8
 4354        movdqa	xmm13, xmm8
 4355        movdqa	xmm14, xmm8
 4356        pslld	xmm12, 31
 4357        pslld	xmm13, 30
 4358        pslld	xmm14, 25
 4359        pxor	xmm12, xmm13
 4360        pxor	xmm12, xmm14
 4361        movdqa	xmm13, xmm12
 4362        psrldq	xmm13, 4
 4363        pslldq	xmm12, 12
 4364        pxor	xmm8, xmm12
 4365        movdqa	xmm14, xmm8
 4366        movdqa	xmm10, xmm8
 4367        movdqa	xmm9, xmm8
 4368        psrld	xmm14, 1
 4369        psrld	xmm10, 2
 4370        psrld	xmm9, 7
 4371        pxor	xmm14, xmm10
 4372        pxor	xmm14, xmm9
 4373        pxor	xmm14, xmm13
 4374        pxor	xmm14, xmm8
 4375        pxor	xmm3, xmm14
 4376        movdqu	OWORD PTR [rsp+48], xmm3
 4377        ; H ^ 5
 4378        pshufd	xmm9, xmm0, 78
 4379        pshufd	xmm10, xmm1, 78
 4380        movdqa	xmm11, xmm1
 4381        movdqa	xmm8, xmm1
 4382        pclmulqdq	xmm11, xmm0, 17
 4383        pclmulqdq	xmm8, xmm0, 0
 4384        pxor	xmm9, xmm0
 4385        pxor	xmm10, xmm1
 4386        pclmulqdq	xmm9, xmm10, 0
 4387        pxor	xmm9, xmm8
 4388        pxor	xmm9, xmm11
 4389        movdqa	xmm10, xmm9
 4390        movdqa	xmm7, xmm11
 4391        pslldq	xmm10, 8
 4392        psrldq	xmm9, 8
 4393        pxor	xmm8, xmm10
 4394        pxor	xmm7, xmm9
 4395        movdqa	xmm12, xmm8
 4396        movdqa	xmm13, xmm8
 4397        movdqa	xmm14, xmm8
 4398        pslld	xmm12, 31
 4399        pslld	xmm13, 30
 4400        pslld	xmm14, 25
 4401        pxor	xmm12, xmm13
 4402        pxor	xmm12, xmm14
 4403        movdqa	xmm13, xmm12
 4404        psrldq	xmm13, 4
 4405        pslldq	xmm12, 12
 4406        pxor	xmm8, xmm12
 4407        movdqa	xmm14, xmm8
 4408        movdqa	xmm10, xmm8
 4409        movdqa	xmm9, xmm8
 4410        psrld	xmm14, 1
 4411        psrld	xmm10, 2
 4412        psrld	xmm9, 7
 4413        pxor	xmm14, xmm10
 4414        pxor	xmm14, xmm9
 4415        pxor	xmm14, xmm13
 4416        pxor	xmm14, xmm8
 4417        pxor	xmm7, xmm14
 4418        movdqu	OWORD PTR [rsp+64], xmm7
 4419        ; H ^ 6
 4420        pshufd	xmm9, xmm1, 78
 4421        pshufd	xmm10, xmm1, 78
 4422        movdqa	xmm11, xmm1
 4423        movdqa	xmm8, xmm1
 4424        pclmulqdq	xmm11, xmm1, 17
 4425        pclmulqdq	xmm8, xmm1, 0
 4426        pxor	xmm9, xmm1
 4427        pxor	xmm10, xmm1
 4428        pclmulqdq	xmm9, xmm10, 0
 4429        pxor	xmm9, xmm8
 4430        pxor	xmm9, xmm11
 4431        movdqa	xmm10, xmm9
 4432        movdqa	xmm7, xmm11
 4433        pslldq	xmm10, 8
 4434        psrldq	xmm9, 8
 4435        pxor	xmm8, xmm10
 4436        pxor	xmm7, xmm9
 4437        movdqa	xmm12, xmm8
 4438        movdqa	xmm13, xmm8
 4439        movdqa	xmm14, xmm8
 4440        pslld	xmm12, 31
 4441        pslld	xmm13, 30
 4442        pslld	xmm14, 25
 4443        pxor	xmm12, xmm13
 4444        pxor	xmm12, xmm14
 4445        movdqa	xmm13, xmm12
 4446        psrldq	xmm13, 4
 4447        pslldq	xmm12, 12
 4448        pxor	xmm8, xmm12
 4449        movdqa	xmm14, xmm8
 4450        movdqa	xmm10, xmm8
 4451        movdqa	xmm9, xmm8
 4452        psrld	xmm14, 1
 4453        psrld	xmm10, 2
 4454        psrld	xmm9, 7
 4455        pxor	xmm14, xmm10
 4456        pxor	xmm14, xmm9
 4457        pxor	xmm14, xmm13
 4458        pxor	xmm14, xmm8
 4459        pxor	xmm7, xmm14
 4460        movdqu	OWORD PTR [rsp+80], xmm7
 4461        ; H ^ 7
 4462        pshufd	xmm9, xmm1, 78
 4463        pshufd	xmm10, xmm3, 78
 4464        movdqa	xmm11, xmm3
 4465        movdqa	xmm8, xmm3
 4466        pclmulqdq	xmm11, xmm1, 17
 4467        pclmulqdq	xmm8, xmm1, 0
 4468        pxor	xmm9, xmm1
 4469        pxor	xmm10, xmm3
 4470        pclmulqdq	xmm9, xmm10, 0
 4471        pxor	xmm9, xmm8
 4472        pxor	xmm9, xmm11
 4473        movdqa	xmm10, xmm9
 4474        movdqa	xmm7, xmm11
 4475        pslldq	xmm10, 8
 4476        psrldq	xmm9, 8
 4477        pxor	xmm8, xmm10
 4478        pxor	xmm7, xmm9
 4479        movdqa	xmm12, xmm8
 4480        movdqa	xmm13, xmm8
 4481        movdqa	xmm14, xmm8
 4482        pslld	xmm12, 31
 4483        pslld	xmm13, 30
 4484        pslld	xmm14, 25
 4485        pxor	xmm12, xmm13
 4486        pxor	xmm12, xmm14
 4487        movdqa	xmm13, xmm12
 4488        psrldq	xmm13, 4
 4489        pslldq	xmm12, 12
 4490        pxor	xmm8, xmm12
 4491        movdqa	xmm14, xmm8
 4492        movdqa	xmm10, xmm8
 4493        movdqa	xmm9, xmm8
 4494        psrld	xmm14, 1
 4495        psrld	xmm10, 2
 4496        psrld	xmm9, 7
 4497        pxor	xmm14, xmm10
 4498        pxor	xmm14, xmm9
 4499        pxor	xmm14, xmm13
 4500        pxor	xmm14, xmm8
 4501        pxor	xmm7, xmm14
 4502        movdqu	OWORD PTR [rsp+96], xmm7
 4503        ; H ^ 8
 4504        pshufd	xmm9, xmm3, 78
 4505        pshufd	xmm10, xmm3, 78
 4506        movdqa	xmm11, xmm3
 4507        movdqa	xmm8, xmm3
 4508        pclmulqdq	xmm11, xmm3, 17
 4509        pclmulqdq	xmm8, xmm3, 0
 4510        pxor	xmm9, xmm3
 4511        pxor	xmm10, xmm3
 4512        pclmulqdq	xmm9, xmm10, 0
 4513        pxor	xmm9, xmm8
 4514        pxor	xmm9, xmm11
 4515        movdqa	xmm10, xmm9
 4516        movdqa	xmm7, xmm11
 4517        pslldq	xmm10, 8
 4518        psrldq	xmm9, 8
 4519        pxor	xmm8, xmm10
 4520        pxor	xmm7, xmm9
 4521        movdqa	xmm12, xmm8
 4522        movdqa	xmm13, xmm8
 4523        movdqa	xmm14, xmm8
 4524        pslld	xmm12, 31
 4525        pslld	xmm13, 30
 4526        pslld	xmm14, 25
 4527        pxor	xmm12, xmm13
 4528        pxor	xmm12, xmm14
 4529        movdqa	xmm13, xmm12
 4530        psrldq	xmm13, 4
 4531        pslldq	xmm12, 12
 4532        pxor	xmm8, xmm12
 4533        movdqa	xmm14, xmm8
 4534        movdqa	xmm10, xmm8
 4535        movdqa	xmm9, xmm8
 4536        psrld	xmm14, 1
 4537        psrld	xmm10, 2
 4538        psrld	xmm9, 7
 4539        pxor	xmm14, xmm10
 4540        pxor	xmm14, xmm9
 4541        pxor	xmm14, xmm13
 4542        pxor	xmm14, xmm8
 4543        pxor	xmm7, xmm14
 4544        movdqu	OWORD PTR [rsp+112], xmm7
 4545        ; First 128 bytes of input
 4546        movdqu	xmm8, OWORD PTR [r15]
 4547        movdqa	xmm1, OWORD PTR L_aes_gcm_bswap_epi64
 4548        movdqa	xmm0, xmm8
 4549        pshufb	xmm8, xmm1
 4550        movdqa	xmm9, xmm0
 4551        paddd	xmm9, OWORD PTR L_aes_gcm_one
 4552        pshufb	xmm9, xmm1
 4553        movdqa	xmm10, xmm0
 4554        paddd	xmm10, OWORD PTR L_aes_gcm_two
 4555        pshufb	xmm10, xmm1
 4556        movdqa	xmm11, xmm0
 4557        paddd	xmm11, OWORD PTR L_aes_gcm_three
 4558        pshufb	xmm11, xmm1
 4559        movdqa	xmm12, xmm0
 4560        paddd	xmm12, OWORD PTR L_aes_gcm_four
 4561        pshufb	xmm12, xmm1
 4562        movdqa	xmm13, xmm0
 4563        paddd	xmm13, OWORD PTR L_aes_gcm_five
 4564        pshufb	xmm13, xmm1
 4565        movdqa	xmm14, xmm0
 4566        paddd	xmm14, OWORD PTR L_aes_gcm_six
 4567        pshufb	xmm14, xmm1
 4568        movdqa	xmm15, xmm0
 4569        paddd	xmm15, OWORD PTR L_aes_gcm_seven
 4570        pshufb	xmm15, xmm1
 4571        paddd	xmm0, OWORD PTR L_aes_gcm_eight
 4572        movdqa	xmm7, OWORD PTR [rax]
 4573        movdqu	OWORD PTR [r15], xmm0
 4574        pxor	xmm8, xmm7
 4575        pxor	xmm9, xmm7
 4576        pxor	xmm10, xmm7
 4577        pxor	xmm11, xmm7
 4578        pxor	xmm12, xmm7
 4579        pxor	xmm13, xmm7
 4580        pxor	xmm14, xmm7
 4581        pxor	xmm15, xmm7
 4582        movdqa	xmm7, OWORD PTR [rax+16]
 4583        aesenc	xmm8, xmm7
 4584        aesenc	xmm9, xmm7
 4585        aesenc	xmm10, xmm7
 4586        aesenc	xmm11, xmm7
 4587        aesenc	xmm12, xmm7
 4588        aesenc	xmm13, xmm7
 4589        aesenc	xmm14, xmm7
 4590        aesenc	xmm15, xmm7
 4591        movdqa	xmm7, OWORD PTR [rax+32]
 4592        aesenc	xmm8, xmm7
 4593        aesenc	xmm9, xmm7
 4594        aesenc	xmm10, xmm7
 4595        aesenc	xmm11, xmm7
 4596        aesenc	xmm12, xmm7
 4597        aesenc	xmm13, xmm7
 4598        aesenc	xmm14, xmm7
 4599        aesenc	xmm15, xmm7
 4600        movdqa	xmm7, OWORD PTR [rax+48]
 4601        aesenc	xmm8, xmm7
 4602        aesenc	xmm9, xmm7
 4603        aesenc	xmm10, xmm7
 4604        aesenc	xmm11, xmm7
 4605        aesenc	xmm12, xmm7
 4606        aesenc	xmm13, xmm7
 4607        aesenc	xmm14, xmm7
 4608        aesenc	xmm15, xmm7
 4609        movdqa	xmm7, OWORD PTR [rax+64]
 4610        aesenc	xmm8, xmm7
 4611        aesenc	xmm9, xmm7
 4612        aesenc	xmm10, xmm7
 4613        aesenc	xmm11, xmm7
 4614        aesenc	xmm12, xmm7
 4615        aesenc	xmm13, xmm7
 4616        aesenc	xmm14, xmm7
 4617        aesenc	xmm15, xmm7
 4618        movdqa	xmm7, OWORD PTR [rax+80]
 4619        aesenc	xmm8, xmm7
 4620        aesenc	xmm9, xmm7
 4621        aesenc	xmm10, xmm7
 4622        aesenc	xmm11, xmm7
 4623        aesenc	xmm12, xmm7
 4624        aesenc	xmm13, xmm7
 4625        aesenc	xmm14, xmm7
 4626        aesenc	xmm15, xmm7
 4627        movdqa	xmm7, OWORD PTR [rax+96]
 4628        aesenc	xmm8, xmm7
 4629        aesenc	xmm9, xmm7
 4630        aesenc	xmm10, xmm7
 4631        aesenc	xmm11, xmm7
 4632        aesenc	xmm12, xmm7
 4633        aesenc	xmm13, xmm7
 4634        aesenc	xmm14, xmm7
 4635        aesenc	xmm15, xmm7
 4636        movdqa	xmm7, OWORD PTR [rax+112]
 4637        aesenc	xmm8, xmm7
 4638        aesenc	xmm9, xmm7
 4639        aesenc	xmm10, xmm7
 4640        aesenc	xmm11, xmm7
 4641        aesenc	xmm12, xmm7
 4642        aesenc	xmm13, xmm7
 4643        aesenc	xmm14, xmm7
 4644        aesenc	xmm15, xmm7
 4645        movdqa	xmm7, OWORD PTR [rax+128]
 4646        aesenc	xmm8, xmm7
 4647        aesenc	xmm9, xmm7
 4648        aesenc	xmm10, xmm7
 4649        aesenc	xmm11, xmm7
 4650        aesenc	xmm12, xmm7
 4651        aesenc	xmm13, xmm7
 4652        aesenc	xmm14, xmm7
 4653        aesenc	xmm15, xmm7
 4654        movdqa	xmm7, OWORD PTR [rax+144]
 4655        aesenc	xmm8, xmm7
 4656        aesenc	xmm9, xmm7
 4657        aesenc	xmm10, xmm7
 4658        aesenc	xmm11, xmm7
 4659        aesenc	xmm12, xmm7
 4660        aesenc	xmm13, xmm7
 4661        aesenc	xmm14, xmm7
 4662        aesenc	xmm15, xmm7
 4663        cmp	r8d, 11
 4664        movdqa	xmm7, OWORD PTR [rax+160]
 4665        jl	L_AES_GCM_encrypt_update_aesni_enc_done
 4666        aesenc	xmm8, xmm7
 4667        aesenc	xmm9, xmm7
 4668        aesenc	xmm10, xmm7
 4669        aesenc	xmm11, xmm7
 4670        aesenc	xmm12, xmm7
 4671        aesenc	xmm13, xmm7
 4672        aesenc	xmm14, xmm7
 4673        aesenc	xmm15, xmm7
 4674        movdqa	xmm7, OWORD PTR [rax+176]
 4675        aesenc	xmm8, xmm7
 4676        aesenc	xmm9, xmm7
 4677        aesenc	xmm10, xmm7
 4678        aesenc	xmm11, xmm7
 4679        aesenc	xmm12, xmm7
 4680        aesenc	xmm13, xmm7
 4681        aesenc	xmm14, xmm7
 4682        aesenc	xmm15, xmm7
 4683        cmp	r8d, 13
 4684        movdqa	xmm7, OWORD PTR [rax+192]
 4685        jl	L_AES_GCM_encrypt_update_aesni_enc_done
 4686        aesenc	xmm8, xmm7
 4687        aesenc	xmm9, xmm7
 4688        aesenc	xmm10, xmm7
 4689        aesenc	xmm11, xmm7
 4690        aesenc	xmm12, xmm7
 4691        aesenc	xmm13, xmm7
 4692        aesenc	xmm14, xmm7
 4693        aesenc	xmm15, xmm7
 4694        movdqa	xmm7, OWORD PTR [rax+208]
 4695        aesenc	xmm8, xmm7
 4696        aesenc	xmm9, xmm7
 4697        aesenc	xmm10, xmm7
 4698        aesenc	xmm11, xmm7
 4699        aesenc	xmm12, xmm7
 4700        aesenc	xmm13, xmm7
 4701        aesenc	xmm14, xmm7
 4702        aesenc	xmm15, xmm7
 4703        movdqa	xmm7, OWORD PTR [rax+224]
 4704L_AES_GCM_encrypt_update_aesni_enc_done:
 4705        aesenclast	xmm8, xmm7
 4706        aesenclast	xmm9, xmm7
 4707        movdqu	xmm0, OWORD PTR [r11]
 4708        movdqu	xmm1, OWORD PTR [r11+16]
 4709        pxor	xmm8, xmm0
 4710        pxor	xmm9, xmm1
 4711        movdqu	OWORD PTR [r10], xmm8
 4712        movdqu	OWORD PTR [r10+16], xmm9
 4713        aesenclast	xmm10, xmm7
 4714        aesenclast	xmm11, xmm7
 4715        movdqu	xmm0, OWORD PTR [r11+32]
 4716        movdqu	xmm1, OWORD PTR [r11+48]
 4717        pxor	xmm10, xmm0
 4718        pxor	xmm11, xmm1
 4719        movdqu	OWORD PTR [r10+32], xmm10
 4720        movdqu	OWORD PTR [r10+48], xmm11
 4721        aesenclast	xmm12, xmm7
 4722        aesenclast	xmm13, xmm7
 4723        movdqu	xmm0, OWORD PTR [r11+64]
 4724        movdqu	xmm1, OWORD PTR [r11+80]
 4725        pxor	xmm12, xmm0
 4726        pxor	xmm13, xmm1
 4727        movdqu	OWORD PTR [r10+64], xmm12
 4728        movdqu	OWORD PTR [r10+80], xmm13
 4729        aesenclast	xmm14, xmm7
 4730        aesenclast	xmm15, xmm7
 4731        movdqu	xmm0, OWORD PTR [r11+96]
 4732        movdqu	xmm1, OWORD PTR [r11+112]
 4733        pxor	xmm14, xmm0
 4734        pxor	xmm15, xmm1
 4735        movdqu	OWORD PTR [r10+96], xmm14
 4736        movdqu	OWORD PTR [r10+112], xmm15
 4737        cmp	r13d, 128
 4738        mov	edi, 128
 4739        jle	L_AES_GCM_encrypt_update_aesni_end_128
 4740        ; More 128 bytes of input
 4741L_AES_GCM_encrypt_update_aesni_ghash_128:
 4742        lea	rcx, QWORD PTR [r11+rdi]
 4743        lea	rdx, QWORD PTR [r10+rdi]
 4744        movdqu	xmm8, OWORD PTR [r15]
 4745        movdqa	xmm1, OWORD PTR L_aes_gcm_bswap_epi64
 4746        movdqa	xmm0, xmm8
 4747        pshufb	xmm8, xmm1
 4748        movdqa	xmm9, xmm0
 4749        paddd	xmm9, OWORD PTR L_aes_gcm_one
 4750        pshufb	xmm9, xmm1
 4751        movdqa	xmm10, xmm0
 4752        paddd	xmm10, OWORD PTR L_aes_gcm_two
 4753        pshufb	xmm10, xmm1
 4754        movdqa	xmm11, xmm0
 4755        paddd	xmm11, OWORD PTR L_aes_gcm_three
 4756        pshufb	xmm11, xmm1
 4757        movdqa	xmm12, xmm0
 4758        paddd	xmm12, OWORD PTR L_aes_gcm_four
 4759        pshufb	xmm12, xmm1
 4760        movdqa	xmm13, xmm0
 4761        paddd	xmm13, OWORD PTR L_aes_gcm_five
 4762        pshufb	xmm13, xmm1
 4763        movdqa	xmm14, xmm0
 4764        paddd	xmm14, OWORD PTR L_aes_gcm_six
 4765        pshufb	xmm14, xmm1
 4766        movdqa	xmm15, xmm0
 4767        paddd	xmm15, OWORD PTR L_aes_gcm_seven
 4768        pshufb	xmm15, xmm1
 4769        paddd	xmm0, OWORD PTR L_aes_gcm_eight
 4770        movdqa	xmm7, OWORD PTR [rax]
 4771        movdqu	OWORD PTR [r15], xmm0
 4772        pxor	xmm8, xmm7
 4773        pxor	xmm9, xmm7
 4774        pxor	xmm10, xmm7
 4775        pxor	xmm11, xmm7
 4776        pxor	xmm12, xmm7
 4777        pxor	xmm13, xmm7
 4778        pxor	xmm14, xmm7
 4779        pxor	xmm15, xmm7
 4780        movdqu	xmm7, OWORD PTR [rsp+112]
 4781        movdqu	xmm0, OWORD PTR [rdx+-128]
 4782        aesenc	xmm8, [rax+16]
 4783        pshufb	xmm0, OWORD PTR L_aes_gcm_bswap_mask
 4784        pxor	xmm0, xmm2
 4785        pshufd	xmm1, xmm7, 78
 4786        pshufd	xmm5, xmm0, 78
 4787        pxor	xmm1, xmm7
 4788        pxor	xmm5, xmm0
 4789        movdqa	xmm3, xmm0
 4790        pclmulqdq	xmm3, xmm7, 17
 4791        aesenc	xmm9, [rax+16]
 4792        aesenc	xmm10, [rax+16]
 4793        movdqa	xmm2, xmm0
 4794        pclmulqdq	xmm2, xmm7, 0
 4795        aesenc	xmm11, [rax+16]
 4796        aesenc	xmm12, [rax+16]
 4797        pclmulqdq	xmm1, xmm5, 0
 4798        aesenc	xmm13, [rax+16]
 4799        aesenc	xmm14, [rax+16]
 4800        aesenc	xmm15, [rax+16]
 4801        pxor	xmm1, xmm2
 4802        pxor	xmm1, xmm3
 4803        movdqu	xmm7, OWORD PTR [rsp+96]
 4804        movdqu	xmm0, OWORD PTR [rdx+-112]
 4805        pshufd	xmm4, xmm7, 78
 4806        pshufb	xmm0, OWORD PTR L_aes_gcm_bswap_mask
 4807        aesenc	xmm8, [rax+32]
 4808        pxor	xmm4, xmm7
 4809        pshufd	xmm5, xmm0, 78
 4810        pxor	xmm5, xmm0
 4811        movdqa	xmm6, xmm0
 4812        pclmulqdq	xmm6, xmm7, 17
 4813        aesenc	xmm9, [rax+32]
 4814        aesenc	xmm10, [rax+32]
 4815        pclmulqdq	xmm7, xmm0, 0
 4816        aesenc	xmm11, [rax+32]
 4817        aesenc	xmm12, [rax+32]
 4818        pclmulqdq	xmm4, xmm5, 0
 4819        aesenc	xmm13, [rax+32]
 4820        aesenc	xmm14, [rax+32]
 4821        aesenc	xmm15, [rax+32]
 4822        pxor	xmm1, xmm7
 4823        pxor	xmm2, xmm7
 4824        pxor	xmm1, xmm6
 4825        pxor	xmm3, xmm6
 4826        pxor	xmm1, xmm4
 4827        movdqu	xmm7, OWORD PTR [rsp+80]
 4828        movdqu	xmm0, OWORD PTR [rdx+-96]
 4829        pshufd	xmm4, xmm7, 78
 4830        pshufb	xmm0, OWORD PTR L_aes_gcm_bswap_mask
 4831        aesenc	xmm8, [rax+48]
 4832        pxor	xmm4, xmm7
 4833        pshufd	xmm5, xmm0, 78
 4834        pxor	xmm5, xmm0
 4835        movdqa	xmm6, xmm0
 4836        pclmulqdq	xmm6, xmm7, 17
 4837        aesenc	xmm9, [rax+48]
 4838        aesenc	xmm10, [rax+48]
 4839        pclmulqdq	xmm7, xmm0, 0
 4840        aesenc	xmm11, [rax+48]
 4841        aesenc	xmm12, [rax+48]
 4842        pclmulqdq	xmm4, xmm5, 0
 4843        aesenc	xmm13, [rax+48]
 4844        aesenc	xmm14, [rax+48]
 4845        aesenc	xmm15, [rax+48]
 4846        pxor	xmm1, xmm7
 4847        pxor	xmm2, xmm7
 4848        pxor	xmm1, xmm6
 4849        pxor	xmm3, xmm6
 4850        pxor	xmm1, xmm4
 4851        movdqu	xmm7, OWORD PTR [rsp+64]
 4852        movdqu	xmm0, OWORD PTR [rdx+-80]
 4853        pshufd	xmm4, xmm7, 78
 4854        pshufb	xmm0, OWORD PTR L_aes_gcm_bswap_mask
 4855        aesenc	xmm8, [rax+64]
 4856        pxor	xmm4, xmm7
 4857        pshufd	xmm5, xmm0, 78
 4858        pxor	xmm5, xmm0
 4859        movdqa	xmm6, xmm0
 4860        pclmulqdq	xmm6, xmm7, 17
 4861        aesenc	xmm9, [rax+64]
 4862        aesenc	xmm10, [rax+64]
 4863        pclmulqdq	xmm7, xmm0, 0
 4864        aesenc	xmm11, [rax+64]
 4865        aesenc	xmm12, [rax+64]
 4866        pclmulqdq	xmm4, xmm5, 0
 4867        aesenc	xmm13, [rax+64]
 4868        aesenc	xmm14, [rax+64]
 4869        aesenc	xmm15, [rax+64]
 4870        pxor	xmm1, xmm7
 4871        pxor	xmm2, xmm7
 4872        pxor	xmm1, xmm6
 4873        pxor	xmm3, xmm6
 4874        pxor	xmm1, xmm4
 4875        movdqu	xmm7, OWORD PTR [rsp+48]
 4876        movdqu	xmm0, OWORD PTR [rdx+-64]
 4877        pshufd	xmm4, xmm7, 78
 4878        pshufb	xmm0, OWORD PTR L_aes_gcm_bswap_mask
 4879        aesenc	xmm8, [rax+80]
 4880        pxor	xmm4, xmm7
 4881        pshufd	xmm5, xmm0, 78
 4882        pxor	xmm5, xmm0
 4883        movdqa	xmm6, xmm0
 4884        pclmulqdq	xmm6, xmm7, 17
 4885        aesenc	xmm9, [rax+80]
 4886        aesenc	xmm10, [rax+80]
 4887        pclmulqdq	xmm7, xmm0, 0
 4888        aesenc	xmm11, [rax+80]
 4889        aesenc	xmm12, [rax+80]
 4890        pclmulqdq	xmm4, xmm5, 0
 4891        aesenc	xmm13, [rax+80]
 4892        aesenc	xmm14, [rax+80]
 4893        aesenc	xmm15, [rax+80]
 4894        pxor	xmm1, xmm7
 4895        pxor	xmm2, xmm7
 4896        pxor	xmm1, xmm6
 4897        pxor	xmm3, xmm6
 4898        pxor	xmm1, xmm4
 4899        movdqu	xmm7, OWORD PTR [rsp+32]
 4900        movdqu	xmm0, OWORD PTR [rdx+-48]
 4901        pshufd	xmm4, xmm7, 78
 4902        pshufb	xmm0, OWORD PTR L_aes_gcm_bswap_mask
 4903        aesenc	xmm8, [rax+96]
 4904        pxor	xmm4, xmm7
 4905        pshufd	xmm5, xmm0, 78
 4906        pxor	xmm5, xmm0
 4907        movdqa	xmm6, xmm0
 4908        pclmulqdq	xmm6, xmm7, 17
 4909        aesenc	xmm9, [rax+96]
 4910        aesenc	xmm10, [rax+96]
 4911        pclmulqdq	xmm7, xmm0, 0
 4912        aesenc	xmm11, [rax+96]
 4913        aesenc	xmm12, [rax+96]
 4914        pclmulqdq	xmm4, xmm5, 0
 4915        aesenc	xmm13, [rax+96]
 4916        aesenc	xmm14, [rax+96]
 4917        aesenc	xmm15, [rax+96]
 4918        pxor	xmm1, xmm7
 4919        pxor	xmm2, xmm7
 4920        pxor	xmm1, xmm6
 4921        pxor	xmm3, xmm6
 4922        pxor	xmm1, xmm4
 4923        movdqu	xmm7, OWORD PTR [rsp+16]
 4924        movdqu	xmm0, OWORD PTR [rdx+-32]
 4925        pshufd	xmm4, xmm7, 78
 4926        pshufb	xmm0, OWORD PTR L_aes_gcm_bswap_mask
 4927        aesenc	xmm8, [rax+112]
 4928        pxor	xmm4, xmm7
 4929        pshufd	xmm5, xmm0, 78
 4930        pxor	xmm5, xmm0
 4931        movdqa	xmm6, xmm0
 4932        pclmulqdq	xmm6, xmm7, 17
 4933        aesenc	xmm9, [rax+112]
 4934        aesenc	xmm10, [rax+112]
 4935        pclmulqdq	xmm7, xmm0, 0
 4936        aesenc	xmm11, [rax+112]
 4937        aesenc	xmm12, [rax+112]
 4938        pclmulqdq	xmm4, xmm5, 0
 4939        aesenc	xmm13, [rax+112]
 4940        aesenc	xmm14, [rax+112]
 4941        aesenc	xmm15, [rax+112]
 4942        pxor	xmm1, xmm7
 4943        pxor	xmm2, xmm7
 4944        pxor	xmm1, xmm6
 4945        pxor	xmm3, xmm6
 4946        pxor	xmm1, xmm4
 4947        movdqu	xmm7, OWORD PTR [rsp]
 4948        movdqu	xmm0, OWORD PTR [rdx+-16]
 4949        pshufd	xmm4, xmm7, 78
 4950        pshufb	xmm0, OWORD PTR L_aes_gcm_bswap_mask
 4951        aesenc	xmm8, [rax+128]
 4952        pxor	xmm4, xmm7
 4953        pshufd	xmm5, xmm0, 78
 4954        pxor	xmm5, xmm0
 4955        movdqa	xmm6, xmm0
 4956        pclmulqdq	xmm6, xmm7, 17
 4957        aesenc	xmm9, [rax+128]
 4958        aesenc	xmm10, [rax+128]
 4959        pclmulqdq	xmm7, xmm0, 0
 4960        aesenc	xmm11, [rax+128]
 4961        aesenc	xmm12, [rax+128]
 4962        pclmulqdq	xmm4, xmm5, 0
 4963        aesenc	xmm13, [rax+128]
 4964        aesenc	xmm14, [rax+128]
 4965        aesenc	xmm15, [rax+128]
 4966        pxor	xmm1, xmm7
 4967        pxor	xmm2, xmm7
 4968        pxor	xmm1, xmm6
 4969        pxor	xmm3, xmm6
 4970        pxor	xmm1, xmm4
 4971        movdqa	xmm5, xmm1
 4972        psrldq	xmm1, 8
 4973        pslldq	xmm5, 8
 4974        aesenc	xmm8, [rax+144]
 4975        pxor	xmm2, xmm5
 4976        pxor	xmm3, xmm1
 4977        movdqa	xmm7, xmm2
 4978        movdqa	xmm4, xmm2
 4979        movdqa	xmm5, xmm2
 4980        aesenc	xmm9, [rax+144]
 4981        pslld	xmm7, 31
 4982        pslld	xmm4, 30
 4983        pslld	xmm5, 25
 4984        aesenc	xmm10, [rax+144]
 4985        pxor	xmm7, xmm4
 4986        pxor	xmm7, xmm5
 4987        aesenc	xmm11, [rax+144]
 4988        movdqa	xmm4, xmm7
 4989        pslldq	xmm7, 12
 4990        psrldq	xmm4, 4
 4991        aesenc	xmm12, [rax+144]
 4992        pxor	xmm2, xmm7
 4993        movdqa	xmm5, xmm2
 4994        movdqa	xmm1, xmm2
 4995        movdqa	xmm0, xmm2
 4996        aesenc	xmm13, [rax+144]
 4997        psrld	xmm5, 1
 4998        psrld	xmm1, 2
 4999        psrld	xmm0, 7
 5000        aesenc	xmm14, [rax+144]
 5001        pxor	xmm5, xmm1
 5002        pxor	xmm5, xmm0
 5003        aesenc	xmm15, [rax+144]
 5004        pxor	xmm5, xmm4
 5005        pxor	xmm2, xmm5
 5006        pxor	xmm2, xmm3
 5007        cmp	r8d, 11
 5008        movdqa	xmm7, OWORD PTR [rax+160]
 5009        jl	L_AES_GCM_encrypt_update_aesni_aesenc_128_ghash_avx_done
 5010        aesenc	xmm8, xmm7
 5011        aesenc	xmm9, xmm7
 5012        aesenc	xmm10, xmm7
 5013        aesenc	xmm11, xmm7
 5014        aesenc	xmm12, xmm7
 5015        aesenc	xmm13, xmm7
 5016        aesenc	xmm14, xmm7
 5017        aesenc	xmm15, xmm7
 5018        movdqa	xmm7, OWORD PTR [rax+176]
 5019        aesenc	xmm8, xmm7
 5020        aesenc	xmm9, xmm7
 5021        aesenc	xmm10, xmm7
 5022        aesenc	xmm11, xmm7
 5023        aesenc	xmm12, xmm7
 5024        aesenc	xmm13, xmm7
 5025        aesenc	xmm14, xmm7
 5026        aesenc	xmm15, xmm7
 5027        cmp	r8d, 13
 5028        movdqa	xmm7, OWORD PTR [rax+192]
 5029        jl	L_AES_GCM_encrypt_update_aesni_aesenc_128_ghash_avx_done
 5030        aesenc	xmm8, xmm7
 5031        aesenc	xmm9, xmm7
 5032        aesenc	xmm10, xmm7
 5033        aesenc	xmm11, xmm7
 5034        aesenc	xmm12, xmm7
 5035        aesenc	xmm13, xmm7
 5036        aesenc	xmm14, xmm7
 5037        aesenc	xmm15, xmm7
 5038        movdqa	xmm7, OWORD PTR [rax+208]
 5039        aesenc	xmm8, xmm7
 5040        aesenc	xmm9, xmm7
 5041        aesenc	xmm10, xmm7
 5042        aesenc	xmm11, xmm7
 5043        aesenc	xmm12, xmm7
 5044        aesenc	xmm13, xmm7
 5045        aesenc	xmm14, xmm7
 5046        aesenc	xmm15, xmm7
 5047        movdqa	xmm7, OWORD PTR [rax+224]
 5048L_AES_GCM_encrypt_update_aesni_aesenc_128_ghash_avx_done:
 5049        aesenclast	xmm8, xmm7
 5050        aesenclast	xmm9, xmm7
 5051        movdqu	xmm0, OWORD PTR [rcx]
 5052        movdqu	xmm1, OWORD PTR [rcx+16]
 5053        pxor	xmm8, xmm0
 5054        pxor	xmm9, xmm1
 5055        movdqu	OWORD PTR [rdx], xmm8
 5056        movdqu	OWORD PTR [rdx+16], xmm9
 5057        aesenclast	xmm10, xmm7
 5058        aesenclast	xmm11, xmm7
 5059        movdqu	xmm0, OWORD PTR [rcx+32]
 5060        movdqu	xmm1, OWORD PTR [rcx+48]
 5061        pxor	xmm10, xmm0
 5062        pxor	xmm11, xmm1
 5063        movdqu	OWORD PTR [rdx+32], xmm10
 5064        movdqu	OWORD PTR [rdx+48], xmm11
 5065        aesenclast	xmm12, xmm7
 5066        aesenclast	xmm13, xmm7
 5067        movdqu	xmm0, OWORD PTR [rcx+64]
 5068        movdqu	xmm1, OWORD PTR [rcx+80]
 5069        pxor	xmm12, xmm0
 5070        pxor	xmm13, xmm1
 5071        movdqu	OWORD PTR [rdx+64], xmm12
 5072        movdqu	OWORD PTR [rdx+80], xmm13
 5073        aesenclast	xmm14, xmm7
 5074        aesenclast	xmm15, xmm7
 5075        movdqu	xmm0, OWORD PTR [rcx+96]
 5076        movdqu	xmm1, OWORD PTR [rcx+112]
 5077        pxor	xmm14, xmm0
 5078        pxor	xmm15, xmm1
 5079        movdqu	OWORD PTR [rdx+96], xmm14
 5080        movdqu	OWORD PTR [rdx+112], xmm15
 5081        add	edi, 128
 5082        cmp	edi, r13d
 5083        jl	L_AES_GCM_encrypt_update_aesni_ghash_128
 5084L_AES_GCM_encrypt_update_aesni_end_128:
 5085        movdqa	xmm4, OWORD PTR L_aes_gcm_bswap_mask
 5086        pshufb	xmm8, xmm4
 5087        pshufb	xmm9, xmm4
 5088        pshufb	xmm10, xmm4
 5089        pshufb	xmm11, xmm4
 5090        pxor	xmm8, xmm2
 5091        pshufb	xmm12, xmm4
 5092        pshufb	xmm13, xmm4
 5093        pshufb	xmm14, xmm4
 5094        pshufb	xmm15, xmm4
 5095        movdqu	xmm7, OWORD PTR [rsp+112]
 5096        pshufd	xmm1, xmm8, 78
 5097        pshufd	xmm2, xmm7, 78
 5098        movdqa	xmm3, xmm7
 5099        movdqa	xmm0, xmm7
 5100        pclmulqdq	xmm3, xmm8, 17
 5101        pclmulqdq	xmm0, xmm8, 0
 5102        pxor	xmm1, xmm8
 5103        pxor	xmm2, xmm7
 5104        pclmulqdq	xmm1, xmm2, 0
 5105        pxor	xmm1, xmm0
 5106        pxor	xmm1, xmm3
 5107        movdqa	xmm2, xmm1
 5108        movdqa	xmm4, xmm0
 5109        movdqa	xmm6, xmm3
 5110        pslldq	xmm2, 8
 5111        psrldq	xmm1, 8
 5112        pxor	xmm4, xmm2
 5113        pxor	xmm6, xmm1
 5114        movdqu	xmm7, OWORD PTR [rsp+96]
 5115        pshufd	xmm1, xmm9, 78
 5116        pshufd	xmm2, xmm7, 78
 5117        movdqa	xmm3, xmm7
 5118        movdqa	xmm0, xmm7
 5119        pclmulqdq	xmm3, xmm9, 17
 5120        pclmulqdq	xmm0, xmm9, 0
 5121        pxor	xmm1, xmm9
 5122        pxor	xmm2, xmm7
 5123        pclmulqdq	xmm1, xmm2, 0
 5124        pxor	xmm1, xmm0
 5125        pxor	xmm1, xmm3
 5126        movdqa	xmm2, xmm1
 5127        pxor	xmm4, xmm0
 5128        pxor	xmm6, xmm3
 5129        pslldq	xmm2, 8
 5130        psrldq	xmm1, 8
 5131        pxor	xmm4, xmm2
 5132        pxor	xmm6, xmm1
 5133        movdqu	xmm7, OWORD PTR [rsp+80]
 5134        pshufd	xmm1, xmm10, 78
 5135        pshufd	xmm2, xmm7, 78
 5136        movdqa	xmm3, xmm7
 5137        movdqa	xmm0, xmm7
 5138        pclmulqdq	xmm3, xmm10, 17
 5139        pclmulqdq	xmm0, xmm10, 0
 5140        pxor	xmm1, xmm10
 5141        pxor	xmm2, xmm7
 5142        pclmulqdq	xmm1, xmm2, 0
 5143        pxor	xmm1, xmm0
 5144        pxor	xmm1, xmm3
 5145        movdqa	xmm2, xmm1
 5146        pxor	xmm4, xmm0
 5147        pxor	xmm6, xmm3
 5148        pslldq	xmm2, 8
 5149        psrldq	xmm1, 8
 5150        pxor	xmm4, xmm2
 5151        pxor	xmm6, xmm1
 5152        movdqu	xmm7, OWORD PTR [rsp+64]
 5153        pshufd	xmm1, xmm11, 78
 5154        pshufd	xmm2, xmm7, 78
 5155        movdqa	xmm3, xmm7
 5156        movdqa	xmm0, xmm7
 5157        pclmulqdq	xmm3, xmm11, 17
 5158        pclmulqdq	xmm0, xmm11, 0
 5159        pxor	xmm1, xmm11
 5160        pxor	xmm2, xmm7
 5161        pclmulqdq	xmm1, xmm2, 0
 5162        pxor	xmm1, xmm0
 5163        pxor	xmm1, xmm3
 5164        movdqa	xmm2, xmm1
 5165        pxor	xmm4, xmm0
 5166        pxor	xmm6, xmm3
 5167        pslldq	xmm2, 8
 5168        psrldq	xmm1, 8
 5169        pxor	xmm4, xmm2
 5170        pxor	xmm6, xmm1
 5171        movdqu	xmm7, OWORD PTR [rsp+48]
 5172        pshufd	xmm1, xmm12, 78
 5173        pshufd	xmm2, xmm7, 78
 5174        movdqa	xmm3, xmm7
 5175        movdqa	xmm0, xmm7
 5176        pclmulqdq	xmm3, xmm12, 17
 5177        pclmulqdq	xmm0, xmm12, 0
 5178        pxor	xmm1, xmm12
 5179        pxor	xmm2, xmm7
 5180        pclmulqdq	xmm1, xmm2, 0
 5181        pxor	xmm1, xmm0
 5182        pxor	xmm1, xmm3
 5183        movdqa	xmm2, xmm1
 5184        pxor	xmm4, xmm0
 5185        pxor	xmm6, xmm3
 5186        pslldq	xmm2, 8
 5187        psrldq	xmm1, 8
 5188        pxor	xmm4, xmm2
 5189        pxor	xmm6, xmm1
 5190        movdqu	xmm7, OWORD PTR [rsp+32]
 5191        pshufd	xmm1, xmm13, 78
 5192        pshufd	xmm2, xmm7, 78
 5193        movdqa	xmm3, xmm7
 5194        movdqa	xmm0, xmm7
 5195        pclmulqdq	xmm3, xmm13, 17
 5196        pclmulqdq	xmm0, xmm13, 0
 5197        pxor	xmm1, xmm13
 5198        pxor	xmm2, xmm7
 5199        pclmulqdq	xmm1, xmm2, 0
 5200        pxor	xmm1, xmm0
 5201        pxor	xmm1, xmm3
 5202        movdqa	xmm2, xmm1
 5203        pxor	xmm4, xmm0
 5204        pxor	xmm6, xmm3
 5205        pslldq	xmm2, 8
 5206        psrldq	xmm1, 8
 5207        pxor	xmm4, xmm2
 5208        pxor	xmm6, xmm1
 5209        movdqu	xmm7, OWORD PTR [rsp+16]
 5210        pshufd	xmm1, xmm14, 78
 5211        pshufd	xmm2, xmm7, 78
 5212        movdqa	xmm3, xmm7
 5213        movdqa	xmm0, xmm7
 5214        pclmulqdq	xmm3, xmm14, 17
 5215        pclmulqdq	xmm0, xmm14, 0
 5216        pxor	xmm1, xmm14
 5217        pxor	xmm2, xmm7
 5218        pclmulqdq	xmm1, xmm2, 0
 5219        pxor	xmm1, xmm0
 5220        pxor	xmm1, xmm3
 5221        movdqa	xmm2, xmm1
 5222        pxor	xmm4, xmm0
 5223        pxor	xmm6, xmm3
 5224        pslldq	xmm2, 8
 5225        psrldq	xmm1, 8
 5226        pxor	xmm4, xmm2
 5227        pxor	xmm6, xmm1
 5228        movdqu	xmm7, OWORD PTR [rsp]
 5229        pshufd	xmm1, xmm15, 78
 5230        pshufd	xmm2, xmm7, 78
 5231        movdqa	xmm3, xmm7
 5232        movdqa	xmm0, xmm7
 5233        pclmulqdq	xmm3, xmm15, 17
 5234        pclmulqdq	xmm0, xmm15, 0
 5235        pxor	xmm1, xmm15
 5236        pxor	xmm2, xmm7
 5237        pclmulqdq	xmm1, xmm2, 0
 5238        pxor	xmm1, xmm0
 5239        pxor	xmm1, xmm3
 5240        movdqa	xmm2, xmm1
 5241        pxor	xmm4, xmm0
 5242        pxor	xmm6, xmm3
 5243        pslldq	xmm2, 8
 5244        psrldq	xmm1, 8
 5245        pxor	xmm4, xmm2
 5246        pxor	xmm6, xmm1
 5247        movdqa	xmm0, xmm4
 5248        movdqa	xmm1, xmm4
 5249        movdqa	xmm2, xmm4
 5250        pslld	xmm0, 31
 5251        pslld	xmm1, 30
 5252        pslld	xmm2, 25
 5253        pxor	xmm0, xmm1
 5254        pxor	xmm0, xmm2
 5255        movdqa	xmm1, xmm0
 5256        psrldq	xmm1, 4
 5257        pslldq	xmm0, 12
 5258        pxor	xmm4, xmm0
 5259        movdqa	xmm2, xmm4
 5260        movdqa	xmm3, xmm4
 5261        movdqa	xmm0, xmm4
 5262        psrld	xmm2, 1
 5263        psrld	xmm3, 2
 5264        psrld	xmm0, 7
 5265        pxor	xmm2, xmm3
 5266        pxor	xmm2, xmm0
 5267        pxor	xmm2, xmm1
 5268        pxor	xmm2, xmm4
 5269        pxor	xmm6, xmm2
 5270        movdqu	xmm5, OWORD PTR [rsp]
 5271L_AES_GCM_encrypt_update_aesni_done_128:
 5272        mov	edx, r9d
 5273        cmp	edi, edx
 5274        jge	L_AES_GCM_encrypt_update_aesni_done_enc
 5275        mov	r13d, r9d
 5276        and	r13d, 4294967280
 5277        cmp	edi, r13d
 5278        jge	L_AES_GCM_encrypt_update_aesni_last_block_done
 5279        lea	rcx, QWORD PTR [r11+rdi]
 5280        lea	rdx, QWORD PTR [r10+rdi]
 5281        movdqu	xmm8, OWORD PTR [r15]
 5282        movdqa	xmm9, xmm8
 5283        pshufb	xmm8, OWORD PTR L_aes_gcm_bswap_epi64
 5284        paddd	xmm9, OWORD PTR L_aes_gcm_one
 5285        pxor	xmm8, [rax]
 5286        movdqu	OWORD PTR [r15], xmm9
 5287        aesenc	xmm8, [rax+16]
 5288        aesenc	xmm8, [rax+32]
 5289        aesenc	xmm8, [rax+48]
 5290        aesenc	xmm8, [rax+64]
 5291        aesenc	xmm8, [rax+80]
 5292        aesenc	xmm8, [rax+96]
 5293        aesenc	xmm8, [rax+112]
 5294        aesenc	xmm8, [rax+128]
 5295        aesenc	xmm8, [rax+144]
 5296        cmp	r8d, 11
 5297        movdqa	xmm9, OWORD PTR [rax+160]
 5298        jl	L_AES_GCM_encrypt_update_aesni_aesenc_block_aesenc_avx_last
 5299        aesenc	xmm8, xmm9
 5300        aesenc	xmm8, [rax+176]
 5301        cmp	r8d, 13
 5302        movdqa	xmm9, OWORD PTR [rax+192]
 5303        jl	L_AES_GCM_encrypt_update_aesni_aesenc_block_aesenc_avx_last
 5304        aesenc	xmm8, xmm9
 5305        aesenc	xmm8, [rax+208]
 5306        movdqa	xmm9, OWORD PTR [rax+224]
 5307L_AES_GCM_encrypt_update_aesni_aesenc_block_aesenc_avx_last:
 5308        aesenclast	xmm8, xmm9
 5309        movdqu	xmm9, OWORD PTR [rcx]
 5310        pxor	xmm8, xmm9
 5311        movdqu	OWORD PTR [rdx], xmm8
 5312        pshufb	xmm8, OWORD PTR L_aes_gcm_bswap_mask
 5313        pxor	xmm6, xmm8
 5314        add	edi, 16
 5315        cmp	edi, r13d
 5316        jge	L_AES_GCM_encrypt_update_aesni_last_block_ghash
 5317L_AES_GCM_encrypt_update_aesni_last_block_start:
 5318        lea	rcx, QWORD PTR [r11+rdi]
 5319        lea	rdx, QWORD PTR [r10+rdi]
 5320        movdqu	xmm8, OWORD PTR [r15]
 5321        movdqa	xmm9, xmm8
 5322        pshufb	xmm8, OWORD PTR L_aes_gcm_bswap_epi64
 5323        paddd	xmm9, OWORD PTR L_aes_gcm_one
 5324        pxor	xmm8, [rax]
 5325        movdqu	OWORD PTR [r15], xmm9
 5326        movdqa	xmm10, xmm6
 5327        pclmulqdq	xmm10, xmm5, 16
 5328        aesenc	xmm8, [rax+16]
 5329        aesenc	xmm8, [rax+32]
 5330        movdqa	xmm11, xmm6
 5331        pclmulqdq	xmm11, xmm5, 1
 5332        aesenc	xmm8, [rax+48]
 5333        aesenc	xmm8, [rax+64]
 5334        movdqa	xmm12, xmm6
 5335        pclmulqdq	xmm12, xmm5, 0
 5336        aesenc	xmm8, [rax+80]
 5337        movdqa	xmm1, xmm6
 5338        pclmulqdq	xmm1, xmm5, 17
 5339        aesenc	xmm8, [rax+96]
 5340        pxor	xmm10, xmm11
 5341        movdqa	xmm2, xmm10
 5342        psrldq	xmm10, 8
 5343        pslldq	xmm2, 8
 5344        aesenc	xmm8, [rax+112]
 5345        movdqa	xmm3, xmm1
 5346        pxor	xmm2, xmm12
 5347        pxor	xmm3, xmm10
 5348        movdqa	xmm0, OWORD PTR L_aes_gcm_mod2_128
 5349        movdqa	xmm11, xmm2
 5350        pclmulqdq	xmm11, xmm0, 16
 5351        aesenc	xmm8, [rax+128]
 5352        pshufd	xmm10, xmm2, 78
 5353        pxor	xmm10, xmm11
 5354        movdqa	xmm11, xmm10
 5355        pclmulqdq	xmm11, xmm0, 16
 5356        aesenc	xmm8, [rax+144]
 5357        pshufd	xmm6, xmm10, 78
 5358        pxor	xmm6, xmm11
 5359        pxor	xmm6, xmm3
 5360        cmp	r8d, 11
 5361        movdqa	xmm9, OWORD PTR [rax+160]
 5362        jl	L_AES_GCM_encrypt_update_aesni_aesenc_gfmul_last
 5363        aesenc	xmm8, xmm9
 5364        aesenc	xmm8, [rax+176]
 5365        cmp	r8d, 13
 5366        movdqa	xmm9, OWORD PTR [rax+192]
 5367        jl	L_AES_GCM_encrypt_update_aesni_aesenc_gfmul_last
 5368        aesenc	xmm8, xmm9
 5369        aesenc	xmm8, [rax+208]
 5370        movdqa	xmm9, OWORD PTR [rax+224]
 5371L_AES_GCM_encrypt_update_aesni_aesenc_gfmul_last:
 5372        aesenclast	xmm8, xmm9
 5373        movdqu	xmm9, OWORD PTR [rcx]
 5374        pxor	xmm8, xmm9
 5375        movdqu	OWORD PTR [rdx], xmm8
 5376        pshufb	xmm8, OWORD PTR L_aes_gcm_bswap_mask
 5377        pxor	xmm6, xmm8
 5378        add	edi, 16
 5379        cmp	edi, r13d
 5380        jl	L_AES_GCM_encrypt_update_aesni_last_block_start
 5381L_AES_GCM_encrypt_update_aesni_last_block_ghash:
 5382        pshufd	xmm9, xmm5, 78
 5383        pshufd	xmm10, xmm6, 78
 5384        movdqa	xmm11, xmm6
 5385        movdqa	xmm8, xmm6
 5386        pclmulqdq	xmm11, xmm5, 17
 5387        pclmulqdq	xmm8, xmm5, 0
 5388        pxor	xmm9, xmm5
 5389        pxor	xmm10, xmm6
 5390        pclmulqdq	xmm9, xmm10, 0
 5391        pxor	xmm9, xmm8
 5392        pxor	xmm9, xmm11
 5393        movdqa	xmm10, xmm9
 5394        movdqa	xmm6, xmm11
 5395        pslldq	xmm10, 8
 5396        psrldq	xmm9, 8
 5397        pxor	xmm8, xmm10
 5398        pxor	xmm6, xmm9
 5399        movdqa	xmm12, xmm8
 5400        movdqa	xmm13, xmm8
 5401        movdqa	xmm14, xmm8
 5402        pslld	xmm12, 31
 5403        pslld	xmm13, 30
 5404        pslld	xmm14, 25
 5405        pxor	xmm12, xmm13
 5406        pxor	xmm12, xmm14
 5407        movdqa	xmm13, xmm12
 5408        psrldq	xmm13, 4
 5409        pslldq	xmm12, 12
 5410        pxor	xmm8, xmm12
 5411        movdqa	xmm14, xmm8
 5412        movdqa	xmm10, xmm8
 5413        movdqa	xmm9, xmm8
 5414        psrld	xmm14, 1
 5415        psrld	xmm10, 2
 5416        psrld	xmm9, 7
 5417        pxor	xmm14, xmm10
 5418        pxor	xmm14, xmm9
 5419        pxor	xmm14, xmm13
 5420        pxor	xmm14, xmm8
 5421        pxor	xmm6, xmm14
 5422L_AES_GCM_encrypt_update_aesni_last_block_done:
 5423L_AES_GCM_encrypt_update_aesni_done_enc:
 5424        movdqa	OWORD PTR [r12], xmm6
 5425        movdqu	xmm6, OWORD PTR [rsp+160]
 5426        movdqu	xmm7, OWORD PTR [rsp+176]
 5427        movdqu	xmm8, OWORD PTR [rsp+192]
 5428        movdqu	xmm9, OWORD PTR [rsp+208]
 5429        movdqu	xmm10, OWORD PTR [rsp+224]
 5430        movdqu	xmm11, OWORD PTR [rsp+240]
 5431        movdqu	xmm12, OWORD PTR [rsp+256]
 5432        movdqu	xmm13, OWORD PTR [rsp+272]
 5433        movdqu	xmm14, OWORD PTR [rsp+288]
 5434        movdqu	xmm15, OWORD PTR [rsp+304]
 5435        add	rsp, 320
 5436        pop	rdi
 5437        pop	r15
 5438        pop	r14
 5439        pop	r12
 5440        pop	r13
 5441        ret
 5442AES_GCM_encrypt_update_aesni ENDP
 5443_TEXT ENDS
 5444_TEXT SEGMENT READONLY PARA
 5445AES_GCM_encrypt_final_aesni PROC
 5446        push	r13
 5447        push	r12
 5448        push	r14
 5449        mov	rax, rcx
 5450        mov	r10d, r9d
 5451        mov	r9, rdx
 5452        mov	r11d, DWORD PTR [rsp+64]
 5453        mov	r12, QWORD PTR [rsp+72]
 5454        mov	r14, QWORD PTR [rsp+80]
 5455        sub	rsp, 144
 5456        movdqu	OWORD PTR [rsp+16], xmm6
 5457        movdqu	OWORD PTR [rsp+32], xmm7
 5458        movdqu	OWORD PTR [rsp+48], xmm8
 5459        movdqu	OWORD PTR [rsp+64], xmm9
 5460        movdqu	OWORD PTR [rsp+80], xmm10
 5461        movdqu	OWORD PTR [rsp+96], xmm11
 5462        movdqu	OWORD PTR [rsp+112], xmm12
 5463        movdqu	OWORD PTR [rsp+128], xmm13
 5464        movdqa	xmm4, OWORD PTR [rax]
 5465        movdqa	xmm5, OWORD PTR [r12]
 5466        movdqa	xmm6, OWORD PTR [r14]
 5467        movdqa	xmm8, xmm5
 5468        movdqa	xmm7, xmm5
 5469        psrlq	xmm8, 63
 5470        psllq	xmm7, 1
 5471        pslldq	xmm8, 8
 5472        por	xmm7, xmm8
 5473        pshufd	xmm5, xmm5, 255
 5474        psrad	xmm5, 31
 5475        pand	xmm5, OWORD PTR L_aes_gcm_mod2_128
 5476        pxor	xmm5, xmm7
 5477        mov	edx, r10d
 5478        mov	ecx, r11d
 5479        shl	rdx, 3
 5480        shl	rcx, 3
 5481        pinsrq	xmm0, rdx, 0
 5482        pinsrq	xmm0, rcx, 1
 5483        pxor	xmm4, xmm0
 5484        pshufd	xmm8, xmm5, 78
 5485        pshufd	xmm9, xmm4, 78
 5486        movdqa	xmm10, xmm4
 5487        movdqa	xmm7, xmm4
 5488        pclmulqdq	xmm10, xmm5, 17
 5489        pclmulqdq	xmm7, xmm5, 0
 5490        pxor	xmm8, xmm5
 5491        pxor	xmm9, xmm4
 5492        pclmulqdq	xmm8, xmm9, 0
 5493        pxor	xmm8, xmm7
 5494        pxor	xmm8, xmm10
 5495        movdqa	xmm9, xmm8
 5496        movdqa	xmm4, xmm10
 5497        pslldq	xmm9, 8
 5498        psrldq	xmm8, 8
 5499        pxor	xmm7, xmm9
 5500        pxor	xmm4, xmm8
 5501        movdqa	xmm11, xmm7
 5502        movdqa	xmm12, xmm7
 5503        movdqa	xmm13, xmm7
 5504        pslld	xmm11, 31
 5505        pslld	xmm12, 30
 5506        pslld	xmm13, 25
 5507        pxor	xmm11, xmm12
 5508        pxor	xmm11, xmm13
 5509        movdqa	xmm12, xmm11
 5510        psrldq	xmm12, 4
 5511        pslldq	xmm11, 12
 5512        pxor	xmm7, xmm11
 5513        movdqa	xmm13, xmm7
 5514        movdqa	xmm9, xmm7
 5515        movdqa	xmm8, xmm7
 5516        psrld	xmm13, 1
 5517        psrld	xmm9, 2
 5518        psrld	xmm8, 7
 5519        pxor	xmm13, xmm9
 5520        pxor	xmm13, xmm8
 5521        pxor	xmm13, xmm12
 5522        pxor	xmm13, xmm7
 5523        pxor	xmm4, xmm13
 5524        pshufb	xmm4, OWORD PTR L_aes_gcm_bswap_mask
 5525        movdqu	xmm0, xmm6
 5526        pxor	xmm0, xmm4
 5527        cmp	r8d, 16
 5528        je	L_AES_GCM_encrypt_final_aesni_store_tag_16
 5529        xor	rcx, rcx
 5530        movdqu	OWORD PTR [rsp], xmm0
 5531L_AES_GCM_encrypt_final_aesni_store_tag_loop:
 5532        movzx	r13d, BYTE PTR [rsp+rcx]
 5533        mov	BYTE PTR [r9+rcx], r13b
 5534        inc	ecx
 5535        cmp	ecx, r8d
 5536        jne	L_AES_GCM_encrypt_final_aesni_store_tag_loop
 5537        jmp	L_AES_GCM_encrypt_final_aesni_store_tag_done
 5538L_AES_GCM_encrypt_final_aesni_store_tag_16:
 5539        movdqu	OWORD PTR [r9], xmm0
 5540L_AES_GCM_encrypt_final_aesni_store_tag_done:
 5541        movdqu	xmm6, OWORD PTR [rsp+16]
 5542        movdqu	xmm7, OWORD PTR [rsp+32]
 5543        movdqu	xmm8, OWORD PTR [rsp+48]
 5544        movdqu	xmm9, OWORD PTR [rsp+64]
 5545        movdqu	xmm10, OWORD PTR [rsp+80]
 5546        movdqu	xmm11, OWORD PTR [rsp+96]
 5547        movdqu	xmm12, OWORD PTR [rsp+112]
 5548        movdqu	xmm13, OWORD PTR [rsp+128]
 5549        add	rsp, 144
 5550        pop	r14
 5551        pop	r12
 5552        pop	r13
 5553        ret
 5554AES_GCM_encrypt_final_aesni ENDP
 5555_TEXT ENDS
 5556_TEXT SEGMENT READONLY PARA
 5557AES_GCM_decrypt_update_aesni PROC
 5558        push	r13
 5559        push	r12
 5560        push	r14
 5561        push	r15
 5562        push	rdi
 5563        push	rsi
 5564        mov	rax, rcx
 5565        mov	r10, r8
 5566        mov	r8d, edx
 5567        mov	r11, r9
 5568        mov	r9d, DWORD PTR [rsp+88]
 5569        mov	r12, QWORD PTR [rsp+96]
 5570        mov	r14, QWORD PTR [rsp+104]
 5571        mov	r15, QWORD PTR [rsp+112]
 5572        sub	rsp, 328
 5573        movdqu	OWORD PTR [rsp+168], xmm6
 5574        movdqu	OWORD PTR [rsp+184], xmm7
 5575        movdqu	OWORD PTR [rsp+200], xmm8
 5576        movdqu	OWORD PTR [rsp+216], xmm9
 5577        movdqu	OWORD PTR [rsp+232], xmm10
 5578        movdqu	OWORD PTR [rsp+248], xmm11
 5579        movdqu	OWORD PTR [rsp+264], xmm12
 5580        movdqu	OWORD PTR [rsp+280], xmm13
 5581        movdqu	OWORD PTR [rsp+296], xmm14
 5582        movdqu	OWORD PTR [rsp+312], xmm15
 5583        movdqa	xmm6, OWORD PTR [r12]
 5584        movdqa	xmm5, OWORD PTR [r14]
 5585        movdqa	xmm9, xmm5
 5586        movdqa	xmm8, xmm5
 5587        psrlq	xmm9, 63
 5588        psllq	xmm8, 1
 5589        pslldq	xmm9, 8
 5590        por	xmm8, xmm9
 5591        pshufd	xmm5, xmm5, 255
 5592        psrad	xmm5, 31
 5593        pand	xmm5, OWORD PTR L_aes_gcm_mod2_128
 5594        pxor	xmm5, xmm8
 5595        xor	edi, edi
 5596        cmp	r9d, 128
 5597        mov	r13d, r9d
 5598        jl	L_AES_GCM_decrypt_update_aesni_done_128
 5599        and	r13d, 4294967168
 5600        movdqa	xmm2, xmm6
 5601        ; H ^ 1
 5602        movdqu	OWORD PTR [rsp], xmm5
 5603        ; H ^ 2
 5604        pshufd	xmm9, xmm5, 78
 5605        pshufd	xmm10, xmm5, 78
 5606        movdqa	xmm11, xmm5
 5607        movdqa	xmm8, xmm5
 5608        pclmulqdq	xmm11, xmm5, 17
 5609        pclmulqdq	xmm8, xmm5, 0
 5610        pxor	xmm9, xmm5
 5611        pxor	xmm10, xmm5
 5612        pclmulqdq	xmm9, xmm10, 0
 5613        pxor	xmm9, xmm8
 5614        pxor	xmm9, xmm11
 5615        movdqa	xmm10, xmm9
 5616        movdqa	xmm0, xmm11
 5617        pslldq	xmm10, 8
 5618        psrldq	xmm9, 8
 5619        pxor	xmm8, xmm10
 5620        pxor	xmm0, xmm9
 5621        movdqa	xmm12, xmm8
 5622        movdqa	xmm13, xmm8
 5623        movdqa	xmm14, xmm8
 5624        pslld	xmm12, 31
 5625        pslld	xmm13, 30
 5626        pslld	xmm14, 25
 5627        pxor	xmm12, xmm13
 5628        pxor	xmm12, xmm14
 5629        movdqa	xmm13, xmm12
 5630        psrldq	xmm13, 4
 5631        pslldq	xmm12, 12
 5632        pxor	xmm8, xmm12
 5633        movdqa	xmm14, xmm8
 5634        movdqa	xmm10, xmm8
 5635        movdqa	xmm9, xmm8
 5636        psrld	xmm14, 1
 5637        psrld	xmm10, 2
 5638        psrld	xmm9, 7
 5639        pxor	xmm14, xmm10
 5640        pxor	xmm14, xmm9
 5641        pxor	xmm14, xmm13
 5642        pxor	xmm14, xmm8
 5643        pxor	xmm0, xmm14
 5644        movdqu	OWORD PTR [rsp+16], xmm0
 5645        ; H ^ 3
 5646        pshufd	xmm9, xmm5, 78
 5647        pshufd	xmm10, xmm0, 78
 5648        movdqa	xmm11, xmm0
 5649        movdqa	xmm8, xmm0
 5650        pclmulqdq	xmm11, xmm5, 17
 5651        pclmulqdq	xmm8, xmm5, 0
 5652        pxor	xmm9, xmm5
 5653        pxor	xmm10, xmm0
 5654        pclmulqdq	xmm9, xmm10, 0
 5655        pxor	xmm9, xmm8
 5656        pxor	xmm9, xmm11
 5657        movdqa	xmm10, xmm9
 5658        movdqa	xmm1, xmm11
 5659        pslldq	xmm10, 8
 5660        psrldq	xmm9, 8
 5661        pxor	xmm8, xmm10
 5662        pxor	xmm1, xmm9
 5663        movdqa	xmm12, xmm8
 5664        movdqa	xmm13, xmm8
 5665        movdqa	xmm14, xmm8
 5666        pslld	xmm12, 31
 5667        pslld	xmm13, 30
 5668        pslld	xmm14, 25
 5669        pxor	xmm12, xmm13
 5670        pxor	xmm12, xmm14
 5671        movdqa	xmm13, xmm12
 5672        psrldq	xmm13, 4
 5673        pslldq	xmm12, 12
 5674        pxor	xmm8, xmm12
 5675        movdqa	xmm14, xmm8
 5676        movdqa	xmm10, xmm8
 5677        movdqa	xmm9, xmm8
 5678        psrld	xmm14, 1
 5679        psrld	xmm10, 2
 5680        psrld	xmm9, 7
 5681        pxor	xmm14, xmm10
 5682        pxor	xmm14, xmm9
 5683        pxor	xmm14, xmm13
 5684        pxor	xmm14, xmm8
 5685        pxor	xmm1, xmm14
 5686        movdqu	OWORD PTR [rsp+32], xmm1
 5687        ; H ^ 4
 5688        pshufd	xmm9, xmm0, 78
 5689        pshufd	xmm10, xmm0, 78
 5690        movdqa	xmm11, xmm0
 5691        movdqa	xmm8, xmm0
 5692        pclmulqdq	xmm11, xmm0, 17
 5693        pclmulqdq	xmm8, xmm0, 0
 5694        pxor	xmm9, xmm0
 5695        pxor	xmm10, xmm0
 5696        pclmulqdq	xmm9, xmm10, 0
 5697        pxor	xmm9, xmm8
 5698        pxor	xmm9, xmm11
 5699        movdqa	xmm10, xmm9
 5700        movdqa	xmm3, xmm11
 5701        pslldq	xmm10, 8
 5702        psrldq	xmm9, 8
 5703        pxor	xmm8, xmm10
 5704        pxor	xmm3, xmm9
 5705        movdqa	xmm12, xmm8
 5706        movdqa	xmm13, xmm8
 5707        movdqa	xmm14, xmm8
 5708        pslld	xmm12, 31
 5709        pslld	xmm13, 30
 5710        pslld	xmm14, 25
 5711        pxor	xmm12, xmm13
 5712        pxor	xmm12, xmm14
 5713        movdqa	xmm13, xmm12
 5714        psrldq	xmm13, 4
 5715        pslldq	xmm12, 12
 5716        pxor	xmm8, xmm12
 5717        movdqa	xmm14, xmm8
 5718        movdqa	xmm10, xmm8
 5719        movdqa	xmm9, xmm8
 5720        psrld	xmm14, 1
 5721        psrld	xmm10, 2
 5722        psrld	xmm9, 7
 5723        pxor	xmm14, xmm10
 5724        pxor	xmm14, xmm9
 5725        pxor	xmm14, xmm13
 5726        pxor	xmm14, xmm8
 5727        pxor	xmm3, xmm14
 5728        movdqu	OWORD PTR [rsp+48], xmm3
 5729        ; H ^ 5
 5730        pshufd	xmm9, xmm0, 78
 5731        pshufd	xmm10, xmm1, 78
 5732        movdqa	xmm11, xmm1
 5733        movdqa	xmm8, xmm1
 5734        pclmulqdq	xmm11, xmm0, 17
 5735        pclmulqdq	xmm8, xmm0, 0
 5736        pxor	xmm9, xmm0
 5737        pxor	xmm10, xmm1
 5738        pclmulqdq	xmm9, xmm10, 0
 5739        pxor	xmm9, xmm8
 5740        pxor	xmm9, xmm11
 5741        movdqa	xmm10, xmm9
 5742        movdqa	xmm7, xmm11
 5743        pslldq	xmm10, 8
 5744        psrldq	xmm9, 8
 5745        pxor	xmm8, xmm10
 5746        pxor	xmm7, xmm9
 5747        movdqa	xmm12, xmm8
 5748        movdqa	xmm13, xmm8
 5749        movdqa	xmm14, xmm8
 5750        pslld	xmm12, 31
 5751        pslld	xmm13, 30
 5752        pslld	xmm14, 25
 5753        pxor	xmm12, xmm13
 5754        pxor	xmm12, xmm14
 5755        movdqa	xmm13, xmm12
 5756        psrldq	xmm13, 4
 5757        pslldq	xmm12, 12
 5758        pxor	xmm8, xmm12
 5759        movdqa	xmm14, xmm8
 5760        movdqa	xmm10, xmm8
 5761        movdqa	xmm9, xmm8
 5762        psrld	xmm14, 1
 5763        psrld	xmm10, 2
 5764        psrld	xmm9, 7
 5765        pxor	xmm14, xmm10
 5766        pxor	xmm14, xmm9
 5767        pxor	xmm14, xmm13
 5768        pxor	xmm14, xmm8
 5769        pxor	xmm7, xmm14
 5770        movdqu	OWORD PTR [rsp+64], xmm7
 5771        ; H ^ 6
 5772        pshufd	xmm9, xmm1, 78
 5773        pshufd	xmm10, xmm1, 78
 5774        movdqa	xmm11, xmm1
 5775        movdqa	xmm8, xmm1
 5776        pclmulqdq	xmm11, xmm1, 17
 5777        pclmulqdq	xmm8, xmm1, 0
 5778        pxor	xmm9, xmm1
 5779        pxor	xmm10, xmm1
 5780        pclmulqdq	xmm9, xmm10, 0
 5781        pxor	xmm9, xmm8
 5782        pxor	xmm9, xmm11
 5783        movdqa	xmm10, xmm9
 5784        movdqa	xmm7, xmm11
 5785        pslldq	xmm10, 8
 5786        psrldq	xmm9, 8
 5787        pxor	xmm8, xmm10
 5788        pxor	xmm7, xmm9
 5789        movdqa	xmm12, xmm8
 5790        movdqa	xmm13, xmm8
 5791        movdqa	xmm14, xmm8
 5792        pslld	xmm12, 31
 5793        pslld	xmm13, 30
 5794        pslld	xmm14, 25
 5795        pxor	xmm12, xmm13
 5796        pxor	xmm12, xmm14
 5797        movdqa	xmm13, xmm12
 5798        psrldq	xmm13, 4
 5799        pslldq	xmm12, 12
 5800        pxor	xmm8, xmm12
 5801        movdqa	xmm14, xmm8
 5802        movdqa	xmm10, xmm8
 5803        movdqa	xmm9, xmm8
 5804        psrld	xmm14, 1
 5805        psrld	xmm10, 2
 5806        psrld	xmm9, 7
 5807        pxor	xmm14, xmm10
 5808        pxor	xmm14, xmm9
 5809        pxor	xmm14, xmm13
 5810        pxor	xmm14, xmm8
 5811        pxor	xmm7, xmm14
 5812        movdqu	OWORD PTR [rsp+80], xmm7
 5813        ; H ^ 7
 5814        pshufd	xmm9, xmm1, 78
 5815        pshufd	xmm10, xmm3, 78
 5816        movdqa	xmm11, xmm3
 5817        movdqa	xmm8, xmm3
 5818        pclmulqdq	xmm11, xmm1, 17
 5819        pclmulqdq	xmm8, xmm1, 0
 5820        pxor	xmm9, xmm1
 5821        pxor	xmm10, xmm3
 5822        pclmulqdq	xmm9, xmm10, 0
 5823        pxor	xmm9, xmm8
 5824        pxor	xmm9, xmm11
 5825        movdqa	xmm10, xmm9
 5826        movdqa	xmm7, xmm11
 5827        pslldq	xmm10, 8
 5828        psrldq	xmm9, 8
 5829        pxor	xmm8, xmm10
 5830        pxor	xmm7, xmm9
 5831        movdqa	xmm12, xmm8
 5832        movdqa	xmm13, xmm8
 5833        movdqa	xmm14, xmm8
 5834        pslld	xmm12, 31
 5835        pslld	xmm13, 30
 5836        pslld	xmm14, 25
 5837        pxor	xmm12, xmm13
 5838        pxor	xmm12, xmm14
 5839        movdqa	xmm13, xmm12
 5840        psrldq	xmm13, 4
 5841        pslldq	xmm12, 12
 5842        pxor	xmm8, xmm12
 5843        movdqa	xmm14, xmm8
 5844        movdqa	xmm10, xmm8
 5845        movdqa	xmm9, xmm8
 5846        psrld	xmm14, 1
 5847        psrld	xmm10, 2
 5848        psrld	xmm9, 7
 5849        pxor	xmm14, xmm10
 5850        pxor	xmm14, xmm9
 5851        pxor	xmm14, xmm13
 5852        pxor	xmm14, xmm8
 5853        pxor	xmm7, xmm14
 5854        movdqu	OWORD PTR [rsp+96], xmm7
 5855        ; H ^ 8
 5856        pshufd	xmm9, xmm3, 78
 5857        pshufd	xmm10, xmm3, 78
 5858        movdqa	xmm11, xmm3
 5859        movdqa	xmm8, xmm3
 5860        pclmulqdq	xmm11, xmm3, 17
 5861        pclmulqdq	xmm8, xmm3, 0
 5862        pxor	xmm9, xmm3
 5863        pxor	xmm10, xmm3
 5864        pclmulqdq	xmm9, xmm10, 0
 5865        pxor	xmm9, xmm8
 5866        pxor	xmm9, xmm11
 5867        movdqa	xmm10, xmm9
 5868        movdqa	xmm7, xmm11
 5869        pslldq	xmm10, 8
 5870        psrldq	xmm9, 8
 5871        pxor	xmm8, xmm10
 5872        pxor	xmm7, xmm9
 5873        movdqa	xmm12, xmm8
 5874        movdqa	xmm13, xmm8
 5875        movdqa	xmm14, xmm8
 5876        pslld	xmm12, 31
 5877        pslld	xmm13, 30
 5878        pslld	xmm14, 25
 5879        pxor	xmm12, xmm13
 5880        pxor	xmm12, xmm14
 5881        movdqa	xmm13, xmm12
 5882        psrldq	xmm13, 4
 5883        pslldq	xmm12, 12
 5884        pxor	xmm8, xmm12
 5885        movdqa	xmm14, xmm8
 5886        movdqa	xmm10, xmm8
 5887        movdqa	xmm9, xmm8
 5888        psrld	xmm14, 1
 5889        psrld	xmm10, 2
 5890        psrld	xmm9, 7
 5891        pxor	xmm14, xmm10
 5892        pxor	xmm14, xmm9
 5893        pxor	xmm14, xmm13
 5894        pxor	xmm14, xmm8
 5895        pxor	xmm7, xmm14
 5896        movdqu	OWORD PTR [rsp+112], xmm7
 5897L_AES_GCM_decrypt_update_aesni_ghash_128:
 5898        lea	rcx, QWORD PTR [r11+rdi]
 5899        lea	rdx, QWORD PTR [r10+rdi]
 5900        movdqu	xmm8, OWORD PTR [r15]
 5901        movdqa	xmm1, OWORD PTR L_aes_gcm_bswap_epi64
 5902        movdqa	xmm0, xmm8
 5903        pshufb	xmm8, xmm1
 5904        movdqa	xmm9, xmm0
 5905        paddd	xmm9, OWORD PTR L_aes_gcm_one
 5906        pshufb	xmm9, xmm1
 5907        movdqa	xmm10, xmm0
 5908        paddd	xmm10, OWORD PTR L_aes_gcm_two
 5909        pshufb	xmm10, xmm1
 5910        movdqa	xmm11, xmm0
 5911        paddd	xmm11, OWORD PTR L_aes_gcm_three
 5912        pshufb	xmm11, xmm1
 5913        movdqa	xmm12, xmm0
 5914        paddd	xmm12, OWORD PTR L_aes_gcm_four
 5915        pshufb	xmm12, xmm1
 5916        movdqa	xmm13, xmm0
 5917        paddd	xmm13, OWORD PTR L_aes_gcm_five
 5918        pshufb	xmm13, xmm1
 5919        movdqa	xmm14, xmm0
 5920        paddd	xmm14, OWORD PTR L_aes_gcm_six
 5921        pshufb	xmm14, xmm1
 5922        movdqa	xmm15, xmm0
 5923        paddd	xmm15, OWORD PTR L_aes_gcm_seven
 5924        pshufb	xmm15, xmm1
 5925        paddd	xmm0, OWORD PTR L_aes_gcm_eight
 5926        movdqa	xmm7, OWORD PTR [rax]
 5927        movdqu	OWORD PTR [r15], xmm0
 5928        pxor	xmm8, xmm7
 5929        pxor	xmm9, xmm7
 5930        pxor	xmm10, xmm7
 5931        pxor	xmm11, xmm7
 5932        pxor	xmm12, xmm7
 5933        pxor	xmm13, xmm7
 5934        pxor	xmm14, xmm7
 5935        pxor	xmm15, xmm7
 5936        movdqu	xmm7, OWORD PTR [rsp+112]
 5937        movdqu	xmm0, OWORD PTR [rcx]
 5938        aesenc	xmm8, [rax+16]
 5939        pshufb	xmm0, OWORD PTR L_aes_gcm_bswap_mask
 5940        pxor	xmm0, xmm2
 5941        pshufd	xmm1, xmm7, 78
 5942        pshufd	xmm5, xmm0, 78
 5943        pxor	xmm1, xmm7
 5944        pxor	xmm5, xmm0
 5945        movdqa	xmm3, xmm0
 5946        pclmulqdq	xmm3, xmm7, 17
 5947        aesenc	xmm9, [rax+16]
 5948        aesenc	xmm10, [rax+16]
 5949        movdqa	xmm2, xmm0
 5950        pclmulqdq	xmm2, xmm7, 0
 5951        aesenc	xmm11, [rax+16]
 5952        aesenc	xmm12, [rax+16]
 5953        pclmulqdq	xmm1, xmm5, 0
 5954        aesenc	xmm13, [rax+16]
 5955        aesenc	xmm14, [rax+16]
 5956        aesenc	xmm15, [rax+16]
 5957        pxor	xmm1, xmm2
 5958        pxor	xmm1, xmm3
 5959        movdqu	xmm7, OWORD PTR [rsp+96]
 5960        movdqu	xmm0, OWORD PTR [rcx+16]
 5961        pshufd	xmm4, xmm7, 78
 5962        pshufb	xmm0, OWORD PTR L_aes_gcm_bswap_mask
 5963        aesenc	xmm8, [rax+32]
 5964        pxor	xmm4, xmm7
 5965        pshufd	xmm5, xmm0, 78
 5966        pxor	xmm5, xmm0
 5967        movdqa	xmm6, xmm0
 5968        pclmulqdq	xmm6, xmm7, 17
 5969        aesenc	xmm9, [rax+32]
 5970        aesenc	xmm10, [rax+32]
 5971        pclmulqdq	xmm7, xmm0, 0
 5972        aesenc	xmm11, [rax+32]
 5973        aesenc	xmm12, [rax+32]
 5974        pclmulqdq	xmm4, xmm5, 0
 5975        aesenc	xmm13, [rax+32]
 5976        aesenc	xmm14, [rax+32]
 5977        aesenc	xmm15, [rax+32]
 5978        pxor	xmm1, xmm7
 5979        pxor	xmm2, xmm7
 5980        pxor	xmm1, xmm6
 5981        pxor	xmm3, xmm6
 5982        pxor	xmm1, xmm4
 5983        movdqu	xmm7, OWORD PTR [rsp+80]
 5984        movdqu	xmm0, OWORD PTR [rcx+32]
 5985        pshufd	xmm4, xmm7, 78
 5986        pshufb	xmm0, OWORD PTR L_aes_gcm_bswap_mask
 5987        aesenc	xmm8, [rax+48]
 5988        pxor	xmm4, xmm7
 5989        pshufd	xmm5, xmm0, 78
 5990        pxor	xmm5, xmm0
 5991        movdqa	xmm6, xmm0
 5992        pclmulqdq	xmm6, xmm7, 17
 5993        aesenc	xmm9, [rax+48]
 5994        aesenc	xmm10, [rax+48]
 5995        pclmulqdq	xmm7, xmm0, 0
 5996        aesenc	xmm11, [rax+48]
 5997        aesenc	xmm12, [rax+48]
 5998        pclmulqdq	xmm4, xmm5, 0
 5999        aesenc	xmm13, [rax+48]
 6000        aesenc	xmm14, [rax+48]
 6001        aesenc	xmm15, [rax+48]
 6002        pxor	xmm1, xmm7
 6003        pxor	xmm2, xmm7
 6004        pxor	xmm1, xmm6
 6005        pxor	xmm3, xmm6
 6006        pxor	xmm1, xmm4
 6007        movdqu	xmm7, OWORD PTR [rsp+64]
 6008        movdqu	xmm0, OWORD PTR [rcx+48]
 6009        pshufd	xmm4, xmm7, 78
 6010        pshufb	xmm0, OWORD PTR L_aes_gcm_bswap_mask
 6011        aesenc	xmm8, [rax+64]
 6012        pxor	xmm4, xmm7
 6013        pshufd	xmm5, xmm0, 78
 6014        pxor	xmm5, xmm0
 6015        movdqa	xmm6, xmm0
 6016        pclmulqdq	xmm6, xmm7, 17
 6017        aesenc	xmm9, [rax+64]
 6018        aesenc	xmm10, [rax+64]
 6019        pclmulqdq	xmm7, xmm0, 0
 6020        aesenc	xmm11, [rax+64]
 6021        aesenc	xmm12, [rax+64]
 6022        pclmulqdq	xmm4, xmm5, 0
 6023        aesenc	xmm13, [rax+64]
 6024        aesenc	xmm14, [rax+64]
 6025        aesenc	xmm15, [rax+64]
 6026        pxor	xmm1, xmm7
 6027        pxor	xmm2, xmm7
 6028        pxor	xmm1, xmm6
 6029        pxor	xmm3, xmm6
 6030        pxor	xmm1, xmm4
 6031        movdqu	xmm7, OWORD PTR [rsp+48]
 6032        movdqu	xmm0, OWORD PTR [rcx+64]
 6033        pshufd	xmm4, xmm7, 78
 6034        pshufb	xmm0, OWORD PTR L_aes_gcm_bswap_mask
 6035        aesenc	xmm8, [rax+80]
 6036        pxor	xmm4, xmm7
 6037        pshufd	xmm5, xmm0, 78
 6038        pxor	xmm5, xmm0
 6039        movdqa	xmm6, xmm0
 6040        pclmulqdq	xmm6, xmm7, 17
 6041        aesenc	xmm9, [rax+80]
 6042        aesenc	xmm10, [rax+80]
 6043        pclmulqdq	xmm7, xmm0, 0
 6044        aesenc	xmm11, [rax+80]
 6045        aesenc	xmm12, [rax+80]
 6046        pclmulqdq	xmm4, xmm5, 0
 6047        aesenc	xmm13, [rax+80]
 6048        aesenc	xmm14, [rax+80]
 6049        aesenc	xmm15, [rax+80]
 6050        pxor	xmm1, xmm7
 6051        pxor	xmm2, xmm7
 6052        pxor	xmm1, xmm6
 6053        pxor	xmm3, xmm6
 6054        pxor	xmm1, xmm4
 6055        movdqu	xmm7, OWORD PTR [rsp+32]
 6056        movdqu	xmm0, OWORD PTR [rcx+80]
 6057        pshufd	xmm4, xmm7, 78
 6058        pshufb	xmm0, OWORD PTR L_aes_gcm_bswap_mask
 6059        aesenc	xmm8, [rax+96]
 6060        pxor	xmm4, xmm7
 6061        pshufd	xmm5, xmm0, 78
 6062        pxor	xmm5, xmm0
 6063        movdqa	xmm6, xmm0
 6064        pclmulqdq	xmm6, xmm7, 17
 6065        aesenc	xmm9, [rax+96]
 6066        aesenc	xmm10, [rax+96]
 6067        pclmulqdq	xmm7, xmm0, 0
 6068        aesenc	xmm11, [rax+96]
 6069        aesenc	xmm12, [rax+96]
 6070        pclmulqdq	xmm4, xmm5, 0
 6071        aesenc	xmm13, [rax+96]
 6072        aesenc	xmm14, [rax+96]
 6073        aesenc	xmm15, [rax+96]
 6074        pxor	xmm1, xmm7
 6075        pxor	xmm2, xmm7
 6076        pxor	xmm1, xmm6
 6077        pxor	xmm3, xmm6
 6078        pxor	xmm1, xmm4
 6079        movdqu	xmm7, OWORD PTR [rsp+16]
 6080        movdqu	xmm0, OWORD PTR [rcx+96]
 6081        pshufd	xmm4, xmm7, 78
 6082        pshufb	xmm0, OWORD PTR L_aes_gcm_bswap_mask
 6083        aesenc	xmm8, [rax+112]
 6084        pxor	xmm4, xmm7
 6085        pshufd	xmm5, xmm0, 78
 6086        pxor	xmm5, xmm0
 6087        movdqa	xmm6, xmm0
 6088        pclmulqdq	xmm6, xmm7, 17
 6089        aesenc	xmm9, [rax+112]
 6090        aesenc	xmm10, [rax+112]
 6091        pclmulqdq	xmm7, xmm0, 0
 6092        aesenc	xmm11, [rax+112]
 6093        aesenc	xmm12, [rax+112]
 6094        pclmulqdq	xmm4, xmm5, 0
 6095        aesenc	xmm13, [rax+112]
 6096        aesenc	xmm14, [rax+112]
 6097        aesenc	xmm15, [rax+112]
 6098        pxor	xmm1, xmm7
 6099        pxor	xmm2, xmm7
 6100        pxor	xmm1, xmm6
 6101        pxor	xmm3, xmm6
 6102        pxor	xmm1, xmm4
 6103        movdqu	xmm7, OWORD PTR [rsp]
 6104        movdqu	xmm0, OWORD PTR [rcx+112]
 6105        pshufd	xmm4, xmm7, 78
 6106        pshufb	xmm0, OWORD PTR L_aes_gcm_bswap_mask
 6107        aesenc	xmm8, [rax+128]
 6108        pxor	xmm4, xmm7
 6109        pshufd	xmm5, xmm0, 78
 6110        pxor	xmm5, xmm0
 6111        movdqa	xmm6, xmm0
 6112        pclmulqdq	xmm6, xmm7, 17
 6113        aesenc	xmm9, [rax+128]
 6114        aesenc	xmm10, [rax+128]
 6115        pclmulqdq	xmm7, xmm0, 0
 6116        aesenc	xmm11, [rax+128]
 6117        aesenc	xmm12, [rax+128]
 6118        pclmulqdq	xmm4, xmm5, 0
 6119        aesenc	xmm13, [rax+128]
 6120        aesenc	xmm14, [rax+128]
 6121        aesenc	xmm15, [rax+128]
 6122        pxor	xmm1, xmm7
 6123        pxor	xmm2, xmm7
 6124        pxor	xmm1, xmm6
 6125        pxor	xmm3, xmm6
 6126        pxor	xmm1, xmm4
 6127        movdqa	xmm5, xmm1
 6128        psrldq	xmm1, 8
 6129        pslldq	xmm5, 8
 6130        aesenc	xmm8, [rax+144]
 6131        pxor	xmm2, xmm5
 6132        pxor	xmm3, xmm1
 6133        movdqa	xmm7, xmm2
 6134        movdqa	xmm4, xmm2
 6135        movdqa	xmm5, xmm2
 6136        aesenc	xmm9, [rax+144]
 6137        pslld	xmm7, 31
 6138        pslld	xmm4, 30
 6139        pslld	xmm5, 25
 6140        aesenc	xmm10, [rax+144]
 6141        pxor	xmm7, xmm4
 6142        pxor	xmm7, xmm5
 6143        aesenc	xmm11, [rax+144]
 6144        movdqa	xmm4, xmm7
 6145        pslldq	xmm7, 12
 6146        psrldq	xmm4, 4
 6147        aesenc	xmm12, [rax+144]
 6148        pxor	xmm2, xmm7
 6149        movdqa	xmm5, xmm2
 6150        movdqa	xmm1, xmm2
 6151        movdqa	xmm0, xmm2
 6152        aesenc	xmm13, [rax+144]
 6153        psrld	xmm5, 1
 6154        psrld	xmm1, 2
 6155        psrld	xmm0, 7
 6156        aesenc	xmm14, [rax+144]
 6157        pxor	xmm5, xmm1
 6158        pxor	xmm5, xmm0
 6159        aesenc	xmm15, [rax+144]
 6160        pxor	xmm5, xmm4
 6161        pxor	xmm2, xmm5
 6162        pxor	xmm2, xmm3
 6163        cmp	r8d, 11
 6164        movdqa	xmm7, OWORD PTR [rax+160]
 6165        jl	L_AES_GCM_decrypt_update_aesni_aesenc_128_ghash_avx_done
 6166        aesenc	xmm8, xmm7
 6167        aesenc	xmm9, xmm7
 6168        aesenc	xmm10, xmm7
 6169        aesenc	xmm11, xmm7
 6170        aesenc	xmm12, xmm7
 6171        aesenc	xmm13, xmm7
 6172        aesenc	xmm14, xmm7
 6173        aesenc	xmm15, xmm7
 6174        movdqa	xmm7, OWORD PTR [rax+176]
 6175        aesenc	xmm8, xmm7
 6176        aesenc	xmm9, xmm7
 6177        aesenc	xmm10, xmm7
 6178        aesenc	xmm11, xmm7
 6179        aesenc	xmm12, xmm7
 6180        aesenc	xmm13, xmm7
 6181        aesenc	xmm14, xmm7
 6182        aesenc	xmm15, xmm7
 6183        cmp	r8d, 13
 6184        movdqa	xmm7, OWORD PTR [rax+192]
 6185        jl	L_AES_GCM_decrypt_update_aesni_aesenc_128_ghash_avx_done
 6186        aesenc	xmm8, xmm7
 6187        aesenc	xmm9, xmm7
 6188        aesenc	xmm10, xmm7
 6189        aesenc	xmm11, xmm7
 6190        aesenc	xmm12, xmm7
 6191        aesenc	xmm13, xmm7
 6192        aesenc	xmm14, xmm7
 6193        aesenc	xmm15, xmm7
 6194        movdqa	xmm7, OWORD PTR [rax+208]
 6195        aesenc	xmm8, xmm7
 6196        aesenc	xmm9, xmm7
 6197        aesenc	xmm10, xmm7
 6198        aesenc	xmm11, xmm7
 6199        aesenc	xmm12, xmm7
 6200        aesenc	xmm13, xmm7
 6201        aesenc	xmm14, xmm7
 6202        aesenc	xmm15, xmm7
 6203        movdqa	xmm7, OWORD PTR [rax+224]
 6204L_AES_GCM_decrypt_update_aesni_aesenc_128_ghash_avx_done:
 6205        aesenclast	xmm8, xmm7
 6206        aesenclast	xmm9, xmm7
 6207        movdqu	xmm0, OWORD PTR [rcx]
 6208        movdqu	xmm1, OWORD PTR [rcx+16]
 6209        pxor	xmm8, xmm0
 6210        pxor	xmm9, xmm1
 6211        movdqu	OWORD PTR [rdx], xmm8
 6212        movdqu	OWORD PTR [rdx+16], xmm9
 6213        aesenclast	xmm10, xmm7
 6214        aesenclast	xmm11, xmm7
 6215        movdqu	xmm0, OWORD PTR [rcx+32]
 6216        movdqu	xmm1, OWORD PTR [rcx+48]
 6217        pxor	xmm10, xmm0
 6218        pxor	xmm11, xmm1
 6219        movdqu	OWORD PTR [rdx+32], xmm10
 6220        movdqu	OWORD PTR [rdx+48], xmm11
 6221        aesenclast	xmm12, xmm7
 6222        aesenclast	xmm13, xmm7
 6223        movdqu	xmm0, OWORD PTR [rcx+64]
 6224        movdqu	xmm1, OWORD PTR [rcx+80]
 6225        pxor	xmm12, xmm0
 6226        pxor	xmm13, xmm1
 6227        movdqu	OWORD PTR [rdx+64], xmm12
 6228        movdqu	OWORD PTR [rdx+80], xmm13
 6229        aesenclast	xmm14, xmm7
 6230        aesenclast	xmm15, xmm7
 6231        movdqu	xmm0, OWORD PTR [rcx+96]
 6232        movdqu	xmm1, OWORD PTR [rcx+112]
 6233        pxor	xmm14, xmm0
 6234        pxor	xmm15, xmm1
 6235        movdqu	OWORD PTR [rdx+96], xmm14
 6236        movdqu	OWORD PTR [rdx+112], xmm15
 6237        add	edi, 128
 6238        cmp	edi, r13d
 6239        jl	L_AES_GCM_decrypt_update_aesni_ghash_128
 6240        movdqa	xmm6, xmm2
 6241        movdqu	xmm5, OWORD PTR [rsp]
 6242L_AES_GCM_decrypt_update_aesni_done_128:
 6243        mov	edx, r9d
 6244        cmp	edi, edx
 6245        jge	L_AES_GCM_decrypt_update_aesni_done_dec
 6246        mov	r13d, r9d
 6247        and	r13d, 4294967280
 6248        cmp	edi, r13d
 6249        jge	L_AES_GCM_decrypt_update_aesni_last_block_done
 6250L_AES_GCM_decrypt_update_aesni_last_block_start:
 6251        lea	rcx, QWORD PTR [r11+rdi]
 6252        lea	rdx, QWORD PTR [r10+rdi]
 6253        movdqu	xmm1, OWORD PTR [rcx]
 6254        movdqa	xmm0, xmm5
 6255        pshufb	xmm1, OWORD PTR L_aes_gcm_bswap_mask
 6256        pxor	xmm1, xmm6
 6257        movdqu	xmm8, OWORD PTR [r15]
 6258        movdqa	xmm9, xmm8
 6259        pshufb	xmm8, OWORD PTR L_aes_gcm_bswap_epi64
 6260        paddd	xmm9, OWORD PTR L_aes_gcm_one
 6261        pxor	xmm8, [rax]
 6262        movdqu	OWORD PTR [r15], xmm9
 6263        movdqa	xmm10, xmm1
 6264        pclmulqdq	xmm10, xmm0, 16
 6265        aesenc	xmm8, [rax+16]
 6266        aesenc	xmm8, [rax+32]
 6267        movdqa	xmm11, xmm1
 6268        pclmulqdq	xmm11, xmm0, 1
 6269        aesenc	xmm8, [rax+48]
 6270        aesenc	xmm8, [rax+64]
 6271        movdqa	xmm12, xmm1
 6272        pclmulqdq	xmm12, xmm0, 0
 6273        aesenc	xmm8, [rax+80]
 6274        movdqa	xmm1, xmm1
 6275        pclmulqdq	xmm1, xmm0, 17
 6276        aesenc	xmm8, [rax+96]
 6277        pxor	xmm10, xmm11
 6278        movdqa	xmm2, xmm10
 6279        psrldq	xmm10, 8
 6280        pslldq	xmm2, 8
 6281        aesenc	xmm8, [rax+112]
 6282        movdqa	xmm3, xmm1
 6283        pxor	xmm2, xmm12
 6284        pxor	xmm3, xmm10
 6285        movdqa	xmm0, OWORD PTR L_aes_gcm_mod2_128
 6286        movdqa	xmm11, xmm2
 6287        pclmulqdq	xmm11, xmm0, 16
 6288        aesenc	xmm8, [rax+128]
 6289        pshufd	xmm10, xmm2, 78
 6290        pxor	xmm10, xmm11
 6291        movdqa	xmm11, xmm10
 6292        pclmulqdq	xmm11, xmm0, 16
 6293        aesenc	xmm8, [rax+144]
 6294        pshufd	xmm6, xmm10, 78
 6295        pxor	xmm6, xmm11
 6296        pxor	xmm6, xmm3
 6297        cmp	r8d, 11
 6298        movdqa	xmm9, OWORD PTR [rax+160]
 6299        jl	L_AES_GCM_decrypt_update_aesni_aesenc_gfmul_last
 6300        aesenc	xmm8, xmm9
 6301        aesenc	xmm8, [rax+176]
 6302        cmp	r8d, 13
 6303        movdqa	xmm9, OWORD PTR [rax+192]
 6304        jl	L_AES_GCM_decrypt_update_aesni_aesenc_gfmul_last
 6305        aesenc	xmm8, xmm9
 6306        aesenc	xmm8, [rax+208]
 6307        movdqa	xmm9, OWORD PTR [rax+224]
 6308L_AES_GCM_decrypt_update_aesni_aesenc_gfmul_last:
 6309        aesenclast	xmm8, xmm9
 6310        movdqu	xmm9, OWORD PTR [rcx]
 6311        pxor	xmm8, xmm9
 6312        movdqu	OWORD PTR [rdx], xmm8
 6313        add	edi, 16
 6314        cmp	edi, r13d
 6315        jl	L_AES_GCM_decrypt_update_aesni_last_block_start
 6316L_AES_GCM_decrypt_update_aesni_last_block_done:
 6317L_AES_GCM_decrypt_update_aesni_done_dec:
 6318        movdqa	OWORD PTR [r12], xmm6
 6319        movdqu	xmm6, OWORD PTR [rsp+168]
 6320        movdqu	xmm7, OWORD PTR [rsp+184]
 6321        movdqu	xmm8, OWORD PTR [rsp+200]
 6322        movdqu	xmm9, OWORD PTR [rsp+216]
 6323        movdqu	xmm10, OWORD PTR [rsp+232]
 6324        movdqu	xmm11, OWORD PTR [rsp+248]
 6325        movdqu	xmm12, OWORD PTR [rsp+264]
 6326        movdqu	xmm13, OWORD PTR [rsp+280]
 6327        movdqu	xmm14, OWORD PTR [rsp+296]
 6328        movdqu	xmm15, OWORD PTR [rsp+312]
 6329        add	rsp, 328
 6330        pop	rsi
 6331        pop	rdi
 6332        pop	r15
 6333        pop	r14
 6334        pop	r12
 6335        pop	r13
 6336        ret
 6337AES_GCM_decrypt_update_aesni ENDP
 6338_TEXT ENDS
 6339_TEXT SEGMENT READONLY PARA
 6340AES_GCM_decrypt_final_aesni PROC
 6341        push	r13
 6342        push	r12
 6343        push	r14
 6344        push	rbp
 6345        push	r15
 6346        mov	rax, rcx
 6347        mov	r10d, r9d
 6348        mov	r9, rdx
 6349        mov	r11d, DWORD PTR [rsp+80]
 6350        mov	r12, QWORD PTR [rsp+88]
 6351        mov	r14, QWORD PTR [rsp+96]
 6352        mov	rbp, QWORD PTR [rsp+104]
 6353        sub	rsp, 160
 6354        movdqu	OWORD PTR [rsp+16], xmm6
 6355        movdqu	OWORD PTR [rsp+32], xmm7
 6356        movdqu	OWORD PTR [rsp+48], xmm8
 6357        movdqu	OWORD PTR [rsp+64], xmm9
 6358        movdqu	OWORD PTR [rsp+80], xmm10
 6359        movdqu	OWORD PTR [rsp+96], xmm11
 6360        movdqu	OWORD PTR [rsp+112], xmm12
 6361        movdqu	OWORD PTR [rsp+128], xmm13
 6362        movdqu	OWORD PTR [rsp+144], xmm15
 6363        movdqa	xmm6, OWORD PTR [rax]
 6364        movdqa	xmm5, OWORD PTR [r12]
 6365        movdqa	xmm15, OWORD PTR [r14]
 6366        movdqa	xmm8, xmm5
 6367        movdqa	xmm7, xmm5
 6368        psrlq	xmm8, 63
 6369        psllq	xmm7, 1
 6370        pslldq	xmm8, 8
 6371        por	xmm7, xmm8
 6372        pshufd	xmm5, xmm5, 255
 6373        psrad	xmm5, 31
 6374        pand	xmm5, OWORD PTR L_aes_gcm_mod2_128
 6375        pxor	xmm5, xmm7
 6376        mov	edx, r10d
 6377        mov	ecx, r11d
 6378        shl	rdx, 3
 6379        shl	rcx, 3
 6380        pinsrq	xmm0, rdx, 0
 6381        pinsrq	xmm0, rcx, 1
 6382        pxor	xmm6, xmm0
 6383        pshufd	xmm8, xmm5, 78
 6384        pshufd	xmm9, xmm6, 78
 6385        movdqa	xmm10, xmm6
 6386        movdqa	xmm7, xmm6
 6387        pclmulqdq	xmm10, xmm5, 17
 6388        pclmulqdq	xmm7, xmm5, 0
 6389        pxor	xmm8, xmm5
 6390        pxor	xmm9, xmm6
 6391        pclmulqdq	xmm8, xmm9, 0
 6392        pxor	xmm8, xmm7
 6393        pxor	xmm8, xmm10
 6394        movdqa	xmm9, xmm8
 6395        movdqa	xmm6, xmm10
 6396        pslldq	xmm9, 8
 6397        psrldq	xmm8, 8
 6398        pxor	xmm7, xmm9
 6399        pxor	xmm6, xmm8
 6400        movdqa	xmm11, xmm7
 6401        movdqa	xmm12, xmm7
 6402        movdqa	xmm13, xmm7
 6403        pslld	xmm11, 31
 6404        pslld	xmm12, 30
 6405        pslld	xmm13, 25
 6406        pxor	xmm11, xmm12
 6407        pxor	xmm11, xmm13
 6408        movdqa	xmm12, xmm11
 6409        psrldq	xmm12, 4
 6410        pslldq	xmm11, 12
 6411        pxor	xmm7, xmm11
 6412        movdqa	xmm13, xmm7
 6413        movdqa	xmm9, xmm7
 6414        movdqa	xmm8, xmm7
 6415        psrld	xmm13, 1
 6416        psrld	xmm9, 2
 6417        psrld	xmm8, 7
 6418        pxor	xmm13, xmm9
 6419        pxor	xmm13, xmm8
 6420        pxor	xmm13, xmm12
 6421        pxor	xmm13, xmm7
 6422        pxor	xmm6, xmm13
 6423        pshufb	xmm6, OWORD PTR L_aes_gcm_bswap_mask
 6424        movdqu	xmm0, xmm15
 6425        pxor	xmm0, xmm6
 6426        cmp	r8d, 16
 6427        je	L_AES_GCM_decrypt_final_aesni_cmp_tag_16
 6428        sub	rsp, 16
 6429        xor	rcx, rcx
 6430        xor	r15, r15
 6431        movdqu	OWORD PTR [rsp], xmm0
 6432L_AES_GCM_decrypt_final_aesni_cmp_tag_loop:
 6433        movzx	r13d, BYTE PTR [rsp+rcx]
 6434        xor	r13b, BYTE PTR [r9+rcx]
 6435        or	r15b, r13b
 6436        inc	ecx
 6437        cmp	ecx, r8d
 6438        jne	L_AES_GCM_decrypt_final_aesni_cmp_tag_loop
 6439        cmp	r15b, 0
 6440        sete	r15b
 6441        add	rsp, 16
 6442        xor	rcx, rcx
 6443        jmp	L_AES_GCM_decrypt_final_aesni_cmp_tag_done
 6444L_AES_GCM_decrypt_final_aesni_cmp_tag_16:
 6445        movdqu	xmm1, OWORD PTR [r9]
 6446        pcmpeqb	xmm0, xmm1
 6447        pmovmskb	rdx, xmm0
 6448        ; %%edx == 0xFFFF then return 1 else => return 0
 6449        xor	r15d, r15d
 6450        cmp	edx, 65535
 6451        sete	r15b
 6452L_AES_GCM_decrypt_final_aesni_cmp_tag_done:
 6453        mov	DWORD PTR [rbp], r15d
 6454        movdqu	xmm6, OWORD PTR [rsp+16]
 6455        movdqu	xmm7, OWORD PTR [rsp+32]
 6456        movdqu	xmm8, OWORD PTR [rsp+48]
 6457        movdqu	xmm9, OWORD PTR [rsp+64]
 6458        movdqu	xmm10, OWORD PTR [rsp+80]
 6459        movdqu	xmm11, OWORD PTR [rsp+96]
 6460        movdqu	xmm12, OWORD PTR [rsp+112]
 6461        movdqu	xmm13, OWORD PTR [rsp+128]
 6462        movdqu	xmm15, OWORD PTR [rsp+144]
 6463        add	rsp, 160
 6464        pop	r15
 6465        pop	rbp
 6466        pop	r14
 6467        pop	r12
 6468        pop	r13
 6469        ret
 6470AES_GCM_decrypt_final_aesni ENDP
 6471_TEXT ENDS
 6472IFDEF HAVE_INTEL_AVX1
 6473_DATA SEGMENT
 6474ALIGN 16
 6475L_GCM_generate_m0_avx1_rev8 QWORD \
 6476     08090a0b0c0d0e0fh,  0001020304050607h
 6477ptr_L_GCM_generate_m0_avx1_rev8 QWORD L_GCM_generate_m0_avx1_rev8
 6478_DATA ENDS
 6479_DATA SEGMENT
 6480ALIGN 16
 6481L_GCM_generate_m0_avx1_mod2_128 QWORD \
 6482     0000000000000000h, 0e100000000000000h
 6483ptr_L_GCM_generate_m0_avx1_mod2_128 QWORD L_GCM_generate_m0_avx1_mod2_128
 6484_DATA ENDS
 6485_TEXT SEGMENT READONLY PARA
 6486GCM_generate_m0_avx1 PROC
 6487        sub	rsp, 80
 6488        vmovdqu	OWORD PTR [rsp], xmm6
 6489        vmovdqu	OWORD PTR [rsp+16], xmm7
 6490        vmovdqu	OWORD PTR [rsp+32], xmm8
 6491        vmovdqu	OWORD PTR [rsp+48], xmm9
 6492        vmovdqu	OWORD PTR [rsp+64], xmm10
 6493        vmovdqu	xmm9, OWORD PTR L_GCM_generate_m0_avx1_rev8
 6494        vmovdqu	xmm10, OWORD PTR L_GCM_generate_m0_avx1_mod2_128
 6495        vpxor	xmm8, xmm8, xmm8
 6496        vmovdqu	xmm0, OWORD PTR [rcx]
 6497        vmovdqu	OWORD PTR [rdx], xmm8
 6498        vmovdqu	xmm8, xmm0
 6499        vpshufb	xmm0, xmm0, xmm9
 6500        vpsllq	xmm5, xmm0, 63
 6501        vpsrlq	xmm4, xmm0, 1
 6502        vpslldq	xmm1, xmm5, 8
 6503        vpsrldq	xmm5, xmm5, 8
 6504        vpshufd	xmm1, xmm1, 255
 6505        vpor	xmm4, xmm4, xmm5
 6506        vpsrad	xmm1, xmm1, 31
 6507        vpand	xmm1, xmm1, xmm10
 6508        vpxor	xmm1, xmm1, xmm4
 6509        vpsllq	xmm5, xmm1, 63
 6510        vpsrlq	xmm4, xmm1, 1
 6511        vpslldq	xmm2, xmm5, 8
 6512        vpsrldq	xmm5, xmm5, 8
 6513        vpshufd	xmm2, xmm2, 255
 6514        vpor	xmm4, xmm4, xmm5
 6515        vpsrad	xmm2, xmm2, 31
 6516        vpand	xmm2, xmm2, xmm10
 6517        vpxor	xmm2, xmm2, xmm4
 6518        vpsllq	xmm5, xmm2, 63
 6519        vpsrlq	xmm4, xmm2, 1
 6520        vpslldq	xmm3, xmm5, 8
 6521        vpsrldq	xmm5, xmm5, 8
 6522        vpshufd	xmm3, xmm3, 255
 6523        vpor	xmm4, xmm4, xmm5
 6524        vpsrad	xmm3, xmm3, 31
 6525        vpand	xmm3, xmm3, xmm10
 6526        vpxor	xmm3, xmm3, xmm4
 6527        vpshufb	xmm3, xmm3, xmm9
 6528        vpshufb	xmm2, xmm2, xmm9
 6529        vpshufb	xmm1, xmm1, xmm9
 6530        vpshufb	xmm0, xmm0, xmm9
 6531        vpxor	xmm8, xmm3, xmm2
 6532        vmovdqu	OWORD PTR [rdx+16], xmm3
 6533        vmovdqu	OWORD PTR [rdx+32], xmm2
 6534        vmovdqu	OWORD PTR [rdx+48], xmm8
 6535        vmovdqu	OWORD PTR [rdx+64], xmm1
 6536        vpxor	xmm4, xmm3, xmm1
 6537        vpxor	xmm5, xmm2, xmm1
 6538        vpxor	xmm6, xmm8, xmm1
 6539        vmovdqu	OWORD PTR [rdx+80], xmm4
 6540        vmovdqu	OWORD PTR [rdx+96], xmm5
 6541        vmovdqu	OWORD PTR [rdx+112], xmm6
 6542        vmovdqu	OWORD PTR [rdx+128], xmm0
 6543        vpxor	xmm1, xmm1, xmm0
 6544        vpxor	xmm4, xmm3, xmm0
 6545        vpxor	xmm6, xmm2, xmm0
 6546        vmovdqu	OWORD PTR [rdx+144], xmm4
 6547        vmovdqu	OWORD PTR [rdx+160], xmm6
 6548        vpxor	xmm6, xmm3, xmm6
 6549        vmovdqu	OWORD PTR [rdx+176], xmm6
 6550        vmovdqu	OWORD PTR [rdx+192], xmm1
 6551        vpxor	xmm4, xmm3, xmm1
 6552        vpxor	xmm5, xmm2, xmm1
 6553        vpxor	xmm6, xmm8, xmm1
 6554        vmovdqu	OWORD PTR [rdx+208], xmm4
 6555        vmovdqu	OWORD PTR [rdx+224], xmm5
 6556        vmovdqu	OWORD PTR [rdx+240], xmm6
 6557        vmovdqu	xmm0, OWORD PTR [rdx]
 6558        vmovdqu	xmm1, OWORD PTR [rdx+16]
 6559        vmovdqu	xmm2, OWORD PTR [rdx+32]
 6560        vmovdqu	xmm3, OWORD PTR [rdx+48]
 6561        vpshufb	xmm0, xmm0, xmm9
 6562        vpshufb	xmm1, xmm1, xmm9
 6563        vpshufb	xmm2, xmm2, xmm9
 6564        vpshufb	xmm3, xmm3, xmm9
 6565        vpsllq	xmm4, xmm0, 60
 6566        vpsllq	xmm5, xmm1, 60
 6567        vpsllq	xmm6, xmm2, 60
 6568        vpsllq	xmm7, xmm3, 60
 6569        vpsrlq	xmm0, xmm0, 4
 6570        vpsrlq	xmm1, xmm1, 4
 6571        vpsrlq	xmm2, xmm2, 4
 6572        vpsrlq	xmm3, xmm3, 4
 6573        vpsrldq	xmm4, xmm4, 8
 6574        vpsrldq	xmm5, xmm5, 8
 6575        vpsrldq	xmm6, xmm6, 8
 6576        vpsrldq	xmm7, xmm7, 8
 6577        vpor	xmm0, xmm0, xmm4
 6578        vpor	xmm1, xmm1, xmm5
 6579        vpor	xmm2, xmm2, xmm6
 6580        vpor	xmm3, xmm3, xmm7
 6581        vpshufb	xmm0, xmm0, xmm9
 6582        vpshufb	xmm1, xmm1, xmm9
 6583        vpshufb	xmm2, xmm2, xmm9
 6584        vpshufb	xmm3, xmm3, xmm9
 6585        vmovdqu	OWORD PTR [rdx+256], xmm0
 6586        vmovdqu	OWORD PTR [rdx+272], xmm1
 6587        vmovdqu	OWORD PTR [rdx+288], xmm2
 6588        vmovdqu	OWORD PTR [rdx+304], xmm3
 6589        vmovdqu	xmm0, OWORD PTR [rdx+64]
 6590        vmovdqu	xmm1, OWORD PTR [rdx+80]
 6591        vmovdqu	xmm2, OWORD PTR [rdx+96]
 6592        vmovdqu	xmm3, OWORD PTR [rdx+112]
 6593        vpshufb	xmm0, xmm0, xmm9
 6594        vpshufb	xmm1, xmm1, xmm9
 6595        vpshufb	xmm2, xmm2, xmm9
 6596        vpshufb	xmm3, xmm3, xmm9
 6597        vpsllq	xmm4, xmm0, 60
 6598        vpsllq	xmm5, xmm1, 60
 6599        vpsllq	xmm6, xmm2, 60
 6600        vpsllq	xmm7, xmm3, 60
 6601        vpsrlq	xmm0, xmm0, 4
 6602        vpsrlq	xmm1, xmm1, 4
 6603        vpsrlq	xmm2, xmm2, 4
 6604        vpsrlq	xmm3, xmm3, 4
 6605        vpsrldq	xmm4, xmm4, 8
 6606        vpsrldq	xmm5, xmm5, 8
 6607        vpsrldq	xmm6, xmm6, 8
 6608        vpsrldq	xmm7, xmm7, 8
 6609        vpor	xmm0, xmm0, xmm4
 6610        vpor	xmm1, xmm1, xmm5
 6611        vpor	xmm2, xmm2, xmm6
 6612        vpor	xmm3, xmm3, xmm7
 6613        vpshufb	xmm0, xmm0, xmm9
 6614        vpshufb	xmm1, xmm1, xmm9
 6615        vpshufb	xmm2, xmm2, xmm9
 6616        vpshufb	xmm3, xmm3, xmm9
 6617        vmovdqu	OWORD PTR [rdx+320], xmm0
 6618        vmovdqu	OWORD PTR [rdx+336], xmm1
 6619        vmovdqu	OWORD PTR [rdx+352], xmm2
 6620        vmovdqu	OWORD PTR [rdx+368], xmm3
 6621        vmovdqu	xmm0, OWORD PTR [rdx+128]
 6622        vmovdqu	xmm1, OWORD PTR [rdx+144]
 6623        vmovdqu	xmm2, OWORD PTR [rdx+160]
 6624        vmovdqu	xmm3, OWORD PTR [rdx+176]
 6625        vpshufb	xmm0, xmm0, xmm9
 6626        vpshufb	xmm1, xmm1, xmm9
 6627        vpshufb	xmm2, xmm2, xmm9
 6628        vpshufb	xmm3, xmm3, xmm9
 6629        vpsllq	xmm4, xmm0, 60
 6630        vpsllq	xmm5, xmm1, 60
 6631        vpsllq	xmm6, xmm2, 60
 6632        vpsllq	xmm7, xmm3, 60
 6633        vpsrlq	xmm0, xmm0, 4
 6634        vpsrlq	xmm1, xmm1, 4
 6635        vpsrlq	xmm2, xmm2, 4
 6636        vpsrlq	xmm3, xmm3, 4
 6637        vpsrldq	xmm4, xmm4, 8
 6638        vpsrldq	xmm5, xmm5, 8
 6639        vpsrldq	xmm6, xmm6, 8
 6640        vpsrldq	xmm7, xmm7, 8
 6641        vpor	xmm0, xmm0, xmm4
 6642        vpor	xmm1, xmm1, xmm5
 6643        vpor	xmm2, xmm2, xmm6
 6644        vpor	xmm3, xmm3, xmm7
 6645        vpshufb	xmm0, xmm0, xmm9
 6646        vpshufb	xmm1, xmm1, xmm9
 6647        vpshufb	xmm2, xmm2, xmm9
 6648        vpshufb	xmm3, xmm3, xmm9
 6649        vmovdqu	OWORD PTR [rdx+384], xmm0
 6650        vmovdqu	OWORD PTR [rdx+400], xmm1
 6651        vmovdqu	OWORD PTR [rdx+416], xmm2
 6652        vmovdqu	OWORD PTR [rdx+432], xmm3
 6653        vmovdqu	xmm0, OWORD PTR [rdx+192]
 6654        vmovdqu	xmm1, OWORD PTR [rdx+208]
 6655        vmovdqu	xmm2, OWORD PTR [rdx+224]
 6656        vmovdqu	xmm3, OWORD PTR [rdx+240]
 6657        vpshufb	xmm0, xmm0, xmm9
 6658        vpshufb	xmm1, xmm1, xmm9
 6659        vpshufb	xmm2, xmm2, xmm9
 6660        vpshufb	xmm3, xmm3, xmm9
 6661        vpsllq	xmm4, xmm0, 60
 6662        vpsllq	xmm5, xmm1, 60
 6663        vpsllq	xmm6, xmm2, 60
 6664        vpsllq	xmm7, xmm3, 60
 6665        vpsrlq	xmm0, xmm0, 4
 6666        vpsrlq	xmm1, xmm1, 4
 6667        vpsrlq	xmm2, xmm2, 4
 6668        vpsrlq	xmm3, xmm3, 4
 6669        vpsrldq	xmm4, xmm4, 8
 6670        vpsrldq	xmm5, xmm5, 8
 6671        vpsrldq	xmm6, xmm6, 8
 6672        vpsrldq	xmm7, xmm7, 8
 6673        vpor	xmm0, xmm0, xmm4
 6674        vpor	xmm1, xmm1, xmm5
 6675        vpor	xmm2, xmm2, xmm6
 6676        vpor	xmm3, xmm3, xmm7
 6677        vpshufb	xmm0, xmm0, xmm9
 6678        vpshufb	xmm1, xmm1, xmm9
 6679        vpshufb	xmm2, xmm2, xmm9
 6680        vpshufb	xmm3, xmm3, xmm9
 6681        vmovdqu	OWORD PTR [rdx+448], xmm0
 6682        vmovdqu	OWORD PTR [rdx+464], xmm1
 6683        vmovdqu	OWORD PTR [rdx+480], xmm2
 6684        vmovdqu	OWORD PTR [rdx+496], xmm3
 6685        vmovdqu	xmm6, OWORD PTR [rsp]
 6686        vmovdqu	xmm7, OWORD PTR [rsp+16]
 6687        vmovdqu	xmm8, OWORD PTR [rsp+32]
 6688        vmovdqu	xmm9, OWORD PTR [rsp+48]
 6689        vmovdqu	xmm10, OWORD PTR [rsp+64]
 6690        add	rsp, 80
 6691        ret
 6692GCM_generate_m0_avx1 ENDP
 6693_TEXT ENDS
 6694_DATA SEGMENT
 6695ALIGN 16
 6696L_avx1_aes_gcm_one QWORD \
 6697     0000000000000000h,  0000000000000001h
 6698ptr_L_avx1_aes_gcm_one QWORD L_avx1_aes_gcm_one
 6699_DATA ENDS
 6700_DATA SEGMENT
 6701ALIGN 16
 6702L_avx1_aes_gcm_two QWORD \
 6703     0000000000000000h,  0000000000000002h
 6704ptr_L_avx1_aes_gcm_two QWORD L_avx1_aes_gcm_two
 6705_DATA ENDS
 6706_DATA SEGMENT
 6707ALIGN 16
 6708L_avx1_aes_gcm_three QWORD \
 6709     0000000000000000h,  0000000000000003h
 6710ptr_L_avx1_aes_gcm_three QWORD L_avx1_aes_gcm_three
 6711_DATA ENDS
 6712_DATA SEGMENT
 6713ALIGN 16
 6714L_avx1_aes_gcm_four QWORD \
 6715     0000000000000000h,  0000000000000004h
 6716ptr_L_avx1_aes_gcm_four QWORD L_avx1_aes_gcm_four
 6717_DATA ENDS
 6718_DATA SEGMENT
 6719ALIGN 16
 6720L_avx1_aes_gcm_five QWORD \
 6721     0000000000000000h,  0000000000000005h
 6722ptr_L_avx1_aes_gcm_five QWORD L_avx1_aes_gcm_five
 6723_DATA ENDS
 6724_DATA SEGMENT
 6725ALIGN 16
 6726L_avx1_aes_gcm_six QWORD \
 6727     0000000000000000h,  0000000000000006h
 6728ptr_L_avx1_aes_gcm_six QWORD L_avx1_aes_gcm_six
 6729_DATA ENDS
 6730_DATA SEGMENT
 6731ALIGN 16
 6732L_avx1_aes_gcm_seven QWORD \
 6733     0000000000000000h,  0000000000000007h
 6734ptr_L_avx1_aes_gcm_seven QWORD L_avx1_aes_gcm_seven
 6735_DATA ENDS
 6736_DATA SEGMENT
 6737ALIGN 16
 6738L_avx1_aes_gcm_eight QWORD \
 6739     0000000000000000h,  0000000000000008h
 6740ptr_L_avx1_aes_gcm_eight QWORD L_avx1_aes_gcm_eight
 6741_DATA ENDS
 6742_DATA SEGMENT
 6743ALIGN 16
 6744L_avx1_aes_gcm_bswap_epi64 QWORD \
 6745     0001020304050607h,  08090a0b0c0d0e0fh
 6746ptr_L_avx1_aes_gcm_bswap_epi64 QWORD L_avx1_aes_gcm_bswap_epi64
 6747_DATA ENDS
 6748_DATA SEGMENT
 6749ALIGN 16
 6750L_avx1_aes_gcm_bswap_mask QWORD \
 6751     08090a0b0c0d0e0fh,  0001020304050607h
 6752ptr_L_avx1_aes_gcm_bswap_mask QWORD L_avx1_aes_gcm_bswap_mask
 6753_DATA ENDS
 6754_DATA SEGMENT
 6755ALIGN 16
 6756L_avx1_aes_gcm_mod2_128 QWORD \
 6757     0000000000000001h, 0c200000000000000h
 6758ptr_L_avx1_aes_gcm_mod2_128 QWORD L_avx1_aes_gcm_mod2_128
 6759_DATA ENDS
 6760_TEXT SEGMENT READONLY PARA
 6761AES_GCM_encrypt_avx1 PROC
 6762        push	r13
 6763        push	rdi
 6764        push	rsi
 6765        push	r12
 6766        push	rbx
 6767        push	r14
 6768        push	r15
 6769        mov	rdi, rcx
 6770        mov	rsi, rdx
 6771        mov	r12, r8
 6772        mov	rax, r9
 6773        mov	r8, QWORD PTR [rsp+96]
 6774        mov	r9d, DWORD PTR [rsp+104]
 6775        mov	r11d, DWORD PTR [rsp+112]
 6776        mov	ebx, DWORD PTR [rsp+120]
 6777        mov	r14d, DWORD PTR [rsp+128]
 6778        mov	r15, QWORD PTR [rsp+136]
 6779        mov	r10d, DWORD PTR [rsp+144]
 6780        sub	rsp, 320
 6781        vmovdqu	OWORD PTR [rsp+160], xmm6
 6782        vmovdqu	OWORD PTR [rsp+176], xmm7
 6783        vmovdqu	OWORD PTR [rsp+192], xmm8
 6784        vmovdqu	OWORD PTR [rsp+208], xmm9
 6785        vmovdqu	OWORD PTR [rsp+224], xmm10
 6786        vmovdqu	OWORD PTR [rsp+240], xmm11
 6787        vmovdqu	OWORD PTR [rsp+256], xmm12
 6788        vmovdqu	OWORD PTR [rsp+272], xmm13
 6789        vmovdqu	OWORD PTR [rsp+288], xmm14
 6790        vmovdqu	OWORD PTR [rsp+304], xmm15
 6791        vpxor	xmm4, xmm4, xmm4
 6792        vpxor	xmm6, xmm6, xmm6
 6793        mov	edx, ebx
 6794        cmp	edx, 12
 6795        jne	L_AES_GCM_encrypt_avx1_iv_not_12
 6796        ; # Calculate values when IV is 12 bytes
 6797        ; Set counter based on IV
 6798        mov	ecx, 16777216
 6799        vmovq	xmm4, QWORD PTR [rax]
 6800        vpinsrd	xmm4, xmm4, DWORD PTR [rax+8], 2
 6801        vpinsrd	xmm4, xmm4, ecx, 3
 6802        ; H = Encrypt X(=0) and T = Encrypt counter
 6803        vmovdqa	xmm5, OWORD PTR [r15]
 6804        vpxor	xmm1, xmm4, xmm5
 6805        vmovdqa	xmm7, OWORD PTR [r15+16]
 6806        vaesenc	xmm5, xmm5, xmm7
 6807        vaesenc	xmm1, xmm1, xmm7
 6808        vmovdqa	xmm7, OWORD PTR [r15+32]
 6809        vaesenc	xmm5, xmm5, xmm7
 6810        vaesenc	xmm1, xmm1, xmm7
 6811        vmovdqa	xmm7, OWORD PTR [r15+48]
 6812        vaesenc	xmm5, xmm5, xmm7
 6813        vaesenc	xmm1, xmm1, xmm7
 6814        vmovdqa	xmm7, OWORD PTR [r15+64]
 6815        vaesenc	xmm5, xmm5, xmm7
 6816        vaesenc	xmm1, xmm1, xmm7
 6817        vmovdqa	xmm7, OWORD PTR [r15+80]
 6818        vaesenc	xmm5, xmm5, xmm7
 6819        vaesenc	xmm1, xmm1, xmm7
 6820        vmovdqa	xmm7, OWORD PTR [r15+96]
 6821        vaesenc	xmm5, xmm5, xmm7
 6822        vaesenc	xmm1, xmm1, xmm7
 6823        vmovdqa	xmm7, OWORD PTR [r15+112]
 6824        vaesenc	xmm5, xmm5, xmm7
 6825        vaesenc	xmm1, xmm1, xmm7
 6826        vmovdqa	xmm7, OWORD PTR [r15+128]
 6827        vaesenc	xmm5, xmm5, xmm7
 6828        vaesenc	xmm1, xmm1, xmm7
 6829        vmovdqa	xmm7, OWORD PTR [r15+144]
 6830        vaesenc	xmm5, xmm5, xmm7
 6831        vaesenc	xmm1, xmm1, xmm7
 6832        cmp	r10d, 11
 6833        vmovdqa	xmm7, OWORD PTR [r15+160]
 6834        jl	L_AES_GCM_encrypt_avx1_calc_iv_12_last
 6835        vaesenc	xmm5, xmm5, xmm7
 6836        vaesenc	xmm1, xmm1, xmm7
 6837        vmovdqa	xmm7, OWORD PTR [r15+176]
 6838        vaesenc	xmm5, xmm5, xmm7
 6839        vaesenc	xmm1, xmm1, xmm7
 6840        cmp	r10d, 13
 6841        vmovdqa	xmm7, OWORD PTR [r15+192]
 6842        jl	L_AES_GCM_encrypt_avx1_calc_iv_12_last
 6843        vaesenc	xmm5, xmm5, xmm7
 6844        vaesenc	xmm1, xmm1, xmm7
 6845        vmovdqa	xmm7, OWORD PTR [r15+208]
 6846        vaesenc	xmm5, xmm5, xmm7
 6847        vaesenc	xmm1, xmm1, xmm7
 6848        vmovdqa	xmm7, OWORD PTR [r15+224]
 6849L_AES_GCM_encrypt_avx1_calc_iv_12_last:
 6850        vaesenclast	xmm5, xmm5, xmm7
 6851        vaesenclast	xmm1, xmm1, xmm7
 6852        vpshufb	xmm5, xmm5, OWORD PTR L_avx1_aes_gcm_bswap_mask
 6853        vmovdqu	OWORD PTR [rsp+144], xmm1
 6854        jmp	L_AES_GCM_encrypt_avx1_iv_done
 6855L_AES_GCM_encrypt_avx1_iv_not_12:
 6856        ; Calculate values when IV is not 12 bytes
 6857        ; H = Encrypt X(=0)
 6858        vmovdqa	xmm5, OWORD PTR [r15]
 6859        vaesenc	xmm5, xmm5, [r15+16]
 6860        vaesenc	xmm5, xmm5, [r15+32]
 6861        vaesenc	xmm5, xmm5, [r15+48]
 6862        vaesenc	xmm5, xmm5, [r15+64]
 6863        vaesenc	xmm5, xmm5, [r15+80]
 6864        vaesenc	xmm5, xmm5, [r15+96]
 6865        vaesenc	xmm5, xmm5, [r15+112]
 6866        vaesenc	xmm5, xmm5, [r15+128]
 6867        vaesenc	xmm5, xmm5, [r15+144]
 6868        cmp	r10d, 11
 6869        vmovdqa	xmm9, OWORD PTR [r15+160]
 6870        jl	L_AES_GCM_encrypt_avx1_calc_iv_1_aesenc_avx_last
 6871        vaesenc	xmm5, xmm5, xmm9
 6872        vaesenc	xmm5, xmm5, [r15+176]
 6873        cmp	r10d, 13
 6874        vmovdqa	xmm9, OWORD PTR [r15+192]
 6875        jl	L_AES_GCM_encrypt_avx1_calc_iv_1_aesenc_avx_last
 6876        vaesenc	xmm5, xmm5, xmm9
 6877        vaesenc	xmm5, xmm5, [r15+208]
 6878        vmovdqa	xmm9, OWORD PTR [r15+224]
 6879L_AES_GCM_encrypt_avx1_calc_iv_1_aesenc_avx_last:
 6880        vaesenclast	xmm5, xmm5, xmm9
 6881        vpshufb	xmm5, xmm5, OWORD PTR L_avx1_aes_gcm_bswap_mask
 6882        ; Calc counter
 6883        ; Initialization vector
 6884        cmp	edx, 0
 6885        mov	rcx, 0
 6886        je	L_AES_GCM_encrypt_avx1_calc_iv_done
 6887        cmp	edx, 16
 6888        jl	L_AES_GCM_encrypt_avx1_calc_iv_lt16
 6889        and	edx, 4294967280
 6890L_AES_GCM_encrypt_avx1_calc_iv_16_loop:
 6891        vmovdqu	xmm8, OWORD PTR [rax+rcx]
 6892        vpshufb	xmm8, xmm8, OWORD PTR L_avx1_aes_gcm_bswap_mask
 6893        vpxor	xmm4, xmm4, xmm8
 6894        ; ghash_gfmul_avx
 6895        vpshufd	xmm1, xmm4, 78
 6896        vpshufd	xmm2, xmm5, 78
 6897        vpclmulqdq	xmm3, xmm5, xmm4, 17
 6898        vpclmulqdq	xmm0, xmm5, xmm4, 0
 6899        vpxor	xmm1, xmm1, xmm4
 6900        vpxor	xmm2, xmm2, xmm5
 6901        vpclmulqdq	xmm1, xmm1, xmm2, 0
 6902        vpxor	xmm1, xmm1, xmm0
 6903        vpxor	xmm1, xmm1, xmm3
 6904        vmovdqa	xmm7, xmm0
 6905        vmovdqa	xmm4, xmm3
 6906        vpslldq	xmm2, xmm1, 8
 6907        vpsrldq	xmm1, xmm1, 8
 6908        vpxor	xmm7, xmm7, xmm2
 6909        vpxor	xmm4, xmm4, xmm1
 6910        vpsrld	xmm0, xmm7, 31
 6911        vpsrld	xmm1, xmm4, 31
 6912        vpslld	xmm7, xmm7, 1
 6913        vpslld	xmm4, xmm4, 1
 6914        vpsrldq	xmm2, xmm0, 12
 6915        vpslldq	xmm0, xmm0, 4
 6916        vpslldq	xmm1, xmm1, 4
 6917        vpor	xmm4, xmm4, xmm2
 6918        vpor	xmm7, xmm7, xmm0
 6919        vpor	xmm4, xmm4, xmm1
 6920        vpslld	xmm0, xmm7, 31
 6921        vpslld	xmm1, xmm7, 30
 6922        vpslld	xmm2, xmm7, 25
 6923        vpxor	xmm0, xmm0, xmm1
 6924        vpxor	xmm0, xmm0, xmm2
 6925        vmovdqa	xmm1, xmm0
 6926        vpsrldq	xmm1, xmm1, 4
 6927        vpslldq	xmm0, xmm0, 12
 6928        vpxor	xmm7, xmm7, xmm0
 6929        vpsrld	xmm2, xmm7, 1
 6930        vpsrld	xmm3, xmm7, 2
 6931        vpsrld	xmm0, xmm7, 7
 6932        vpxor	xmm2, xmm2, xmm3
 6933        vpxor	xmm2, xmm2, xmm0
 6934        vpxor	xmm2, xmm2, xmm1
 6935        vpxor	xmm2, xmm2, xmm7
 6936        vpxor	xmm4, xmm4, xmm2
 6937        add	ecx, 16
 6938        cmp	ecx, edx
 6939        jl	L_AES_GCM_encrypt_avx1_calc_iv_16_loop
 6940        mov	edx, ebx
 6941        cmp	ecx, edx
 6942        je	L_AES_GCM_encrypt_avx1_calc_iv_done
 6943L_AES_GCM_encrypt_avx1_calc_iv_lt16:
 6944        sub	rsp, 16
 6945        vpxor	xmm8, xmm8, xmm8
 6946        xor	ebx, ebx
 6947        vmovdqu	OWORD PTR [rsp], xmm8
 6948L_AES_GCM_encrypt_avx1_calc_iv_loop:
 6949        movzx	r13d, BYTE PTR [rax+rcx]
 6950        mov	BYTE PTR [rsp+rbx], r13b
 6951        inc	ecx
 6952        inc	ebx
 6953        cmp	ecx, edx
 6954        jl	L_AES_GCM_encrypt_avx1_calc_iv_loop
 6955        vmovdqu	xmm8, OWORD PTR [rsp]
 6956        add	rsp, 16
 6957        vpshufb	xmm8, xmm8, OWORD PTR L_avx1_aes_gcm_bswap_mask
 6958        vpxor	xmm4, xmm4, xmm8
 6959        ; ghash_gfmul_avx
 6960        vpshufd	xmm1, xmm4, 78
 6961        vpshufd	xmm2, xmm5, 78
 6962        vpclmulqdq	xmm3, xmm5, xmm4, 17
 6963        vpclmulqdq	xmm0, xmm5, xmm4, 0
 6964        vpxor	xmm1, xmm1, xmm4
 6965        vpxor	xmm2, xmm2, xmm5
 6966        vpclmulqdq	xmm1, xmm1, xmm2, 0
 6967        vpxor	xmm1, xmm1, xmm0
 6968        vpxor	xmm1, xmm1, xmm3
 6969        vmovdqa	xmm7, xmm0
 6970        vmovdqa	xmm4, xmm3
 6971        vpslldq	xmm2, xmm1, 8
 6972        vpsrldq	xmm1, xmm1, 8
 6973        vpxor	xmm7, xmm7, xmm2
 6974        vpxor	xmm4, xmm4, xmm1
 6975        vpsrld	xmm0, xmm7, 31
 6976        vpsrld	xmm1, xmm4, 31
 6977        vpslld	xmm7, xmm7, 1
 6978        vpslld	xmm4, xmm4, 1
 6979        vpsrldq	xmm2, xmm0, 12
 6980        vpslldq	xmm0, xmm0, 4
 6981        vpslldq	xmm1, xmm1, 4
 6982        vpor	xmm4, xmm4, xmm2
 6983        vpor	xmm7, xmm7, xmm0
 6984        vpor	xmm4, xmm4, xmm1
 6985        vpslld	xmm0, xmm7, 31
 6986        vpslld	xmm1, xmm7, 30
 6987        vpslld	xmm2, xmm7, 25
 6988        vpxor	xmm0, xmm0, xmm1
 6989        vpxor	xmm0, xmm0, xmm2
 6990        vmovdqa	xmm1, xmm0
 6991        vpsrldq	xmm1, xmm1, 4
 6992        vpslldq	xmm0, xmm0, 12
 6993        vpxor	xmm7, xmm7, xmm0
 6994        vpsrld	xmm2, xmm7, 1
 6995        vpsrld	xmm3, xmm7, 2
 6996        vpsrld	xmm0, xmm7, 7
 6997        vpxor	xmm2, xmm2, xmm3
 6998        vpxor	xmm2, xmm2, xmm0
 6999        vpxor	xmm2, xmm2, xmm1
 7000        vpxor	xmm2, xmm2, xmm7
 7001        vpxor	xmm4, xmm4, xmm2
 7002L_AES_GCM_encrypt_avx1_calc_iv_done:
 7003        ; T = Encrypt counter
 7004        vpxor	xmm0, xmm0, xmm0
 7005        shl	edx, 3
 7006        vmovq	xmm0, rdx
 7007        vpxor	xmm4, xmm4, xmm0
 7008        ; ghash_gfmul_avx
 7009        vpshufd	xmm1, xmm4, 78
 7010        vpshufd	xmm2, xmm5, 78
 7011        vpclmulqdq	xmm3, xmm5, xmm4, 17
 7012        vpclmulqdq	xmm0, xmm5, xmm4, 0
 7013        vpxor	xmm1, xmm1, xmm4
 7014        vpxor	xmm2, xmm2, xmm5
 7015        vpclmulqdq	xmm1, xmm1, xmm2, 0
 7016        vpxor	xmm1, xmm1, xmm0
 7017        vpxor	xmm1, xmm1, xmm3
 7018        vmovdqa	xmm7, xmm0
 7019        vmovdqa	xmm4, xmm3
 7020        vpslldq	xmm2, xmm1, 8
 7021        vpsrldq	xmm1, xmm1, 8
 7022        vpxor	xmm7, xmm7, xmm2
 7023        vpxor	xmm4, xmm4, xmm1
 7024        vpsrld	xmm0, xmm7, 31
 7025        vpsrld	xmm1, xmm4, 31
 7026        vpslld	xmm7, xmm7, 1
 7027        vpslld	xmm4, xmm4, 1
 7028        vpsrldq	xmm2, xmm0, 12
 7029        vpslldq	xmm0, xmm0, 4
 7030        vpslldq	xmm1, xmm1, 4
 7031        vpor	xmm4, xmm4, xmm2
 7032        vpor	xmm7, xmm7, xmm0
 7033        vpor	xmm4, xmm4, xmm1
 7034        vpslld	xmm0, xmm7, 31
 7035        vpslld	xmm1, xmm7, 30
 7036        vpslld	xmm2, xmm7, 25
 7037        vpxor	xmm0, xmm0, xmm1
 7038        vpxor	xmm0, xmm0, xmm2
 7039        vmovdqa	xmm1, xmm0
 7040        vpsrldq	xmm1, xmm1, 4
 7041        vpslldq	xmm0, xmm0, 12
 7042        vpxor	xmm7, xmm7, xmm0
 7043        vpsrld	xmm2, xmm7, 1
 7044        vpsrld	xmm3, xmm7, 2
 7045        vpsrld	xmm0, xmm7, 7
 7046        vpxor	xmm2, xmm2, xmm3
 7047        vpxor	xmm2, xmm2, xmm0
 7048        vpxor	xmm2, xmm2, xmm1
 7049        vpxor	xmm2, xmm2, xmm7
 7050        vpxor	xmm4, xmm4, xmm2
 7051        vpshufb	xmm4, xmm4, OWORD PTR L_avx1_aes_gcm_bswap_mask
 7052        ;   Encrypt counter
 7053        vmovdqa	xmm8, OWORD PTR [r15]
 7054        vpxor	xmm8, xmm8, xmm4
 7055        vaesenc	xmm8, xmm8, [r15+16]
 7056        vaesenc	xmm8, xmm8, [r15+32]
 7057        vaesenc	xmm8, xmm8, [r15+48]
 7058        vaesenc	xmm8, xmm8, [r15+64]
 7059        vaesenc	xmm8, xmm8, [r15+80]
 7060        vaesenc	xmm8, xmm8, [r15+96]
 7061        vaesenc	xmm8, xmm8, [r15+112]
 7062        vaesenc	xmm8, xmm8, [r15+128]
 7063        vaesenc	xmm8, xmm8, [r15+144]
 7064        cmp	r10d, 11
 7065        vmovdqa	xmm9, OWORD PTR [r15+160]
 7066        jl	L_AES_GCM_encrypt_avx1_calc_iv_2_aesenc_avx_last
 7067        vaesenc	xmm8, xmm8, xmm9
 7068        vaesenc	xmm8, xmm8, [r15+176]
 7069        cmp	r10d, 13
 7070        vmovdqa	xmm9, OWORD PTR [r15+192]
 7071        jl	L_AES_GCM_encrypt_avx1_calc_iv_2_aesenc_avx_last
 7072        vaesenc	xmm8, xmm8, xmm9
 7073        vaesenc	xmm8, xmm8, [r15+208]
 7074        vmovdqa	xmm9, OWORD PTR [r15+224]
 7075L_AES_GCM_encrypt_avx1_calc_iv_2_aesenc_avx_last:
 7076        vaesenclast	xmm8, xmm8, xmm9
 7077        vmovdqu	OWORD PTR [rsp+144], xmm8
 7078L_AES_GCM_encrypt_avx1_iv_done:
 7079        ; Additional authentication data
 7080        mov	edx, r11d
 7081        cmp	edx, 0
 7082        je	L_AES_GCM_encrypt_avx1_calc_aad_done
 7083        xor	ecx, ecx
 7084        cmp	edx, 16
 7085        jl	L_AES_GCM_encrypt_avx1_calc_aad_lt16
 7086        and	edx, 4294967280
 7087L_AES_GCM_encrypt_avx1_calc_aad_16_loop:
 7088        vmovdqu	xmm8, OWORD PTR [r12+rcx]
 7089        vpshufb	xmm8, xmm8, OWORD PTR L_avx1_aes_gcm_bswap_mask
 7090        vpxor	xmm6, xmm6, xmm8
 7091        ; ghash_gfmul_avx
 7092        vpshufd	xmm1, xmm6, 78
 7093        vpshufd	xmm2, xmm5, 78
 7094        vpclmulqdq	xmm3, xmm5, xmm6, 17
 7095        vpclmulqdq	xmm0, xmm5, xmm6, 0
 7096        vpxor	xmm1, xmm1, xmm6
 7097        vpxor	xmm2, xmm2, xmm5
 7098        vpclmulqdq	xmm1, xmm1, xmm2, 0
 7099        vpxor	xmm1, xmm1, xmm0
 7100        vpxor	xmm1, xmm1, xmm3
 7101        vmovdqa	xmm7, xmm0
 7102        vmovdqa	xmm6, xmm3
 7103        vpslldq	xmm2, xmm1, 8
 7104        vpsrldq	xmm1, xmm1, 8
 7105        vpxor	xmm7, xmm7, xmm2
 7106        vpxor	xmm6, xmm6, xmm1
 7107        vpsrld	xmm0, xmm7, 31
 7108        vpsrld	xmm1, xmm6, 31
 7109        vpslld	xmm7, xmm7, 1
 7110        vpslld	xmm6, xmm6, 1
 7111        vpsrldq	xmm2, xmm0, 12
 7112        vpslldq	xmm0, xmm0, 4
 7113        vpslldq	xmm1, xmm1, 4
 7114        vpor	xmm6, xmm6, xmm2
 7115        vpor	xmm7, xmm7, xmm0
 7116        vpor	xmm6, xmm6, xmm1
 7117        vpslld	xmm0, xmm7, 31
 7118        vpslld	xmm1, xmm7, 30
 7119        vpslld	xmm2, xmm7, 25
 7120        vpxor	xmm0, xmm0, xmm1
 7121        vpxor	xmm0, xmm0, xmm2
 7122        vmovdqa	xmm1, xmm0
 7123        vpsrldq	xmm1, xmm1, 4
 7124        vpslldq	xmm0, xmm0, 12
 7125        vpxor	xmm7, xmm7, xmm0
 7126        vpsrld	xmm2, xmm7, 1
 7127        vpsrld	xmm3, xmm7, 2
 7128        vpsrld	xmm0, xmm7, 7
 7129        vpxor	xmm2, xmm2, xmm3
 7130        vpxor	xmm2, xmm2, xmm0
 7131        vpxor	xmm2, xmm2, xmm1
 7132        vpxor	xmm2, xmm2, xmm7
 7133        vpxor	xmm6, xmm6, xmm2
 7134        add	ecx, 16
 7135        cmp	ecx, edx
 7136        jl	L_AES_GCM_encrypt_avx1_calc_aad_16_loop
 7137        mov	edx, r11d
 7138        cmp	ecx, edx
 7139        je	L_AES_GCM_encrypt_avx1_calc_aad_done
 7140L_AES_GCM_encrypt_avx1_calc_aad_lt16:
 7141        sub	rsp, 16
 7142        vpxor	xmm8, xmm8, xmm8
 7143        xor	ebx, ebx
 7144        vmovdqu	OWORD PTR [rsp], xmm8
 7145L_AES_GCM_encrypt_avx1_calc_aad_loop:
 7146        movzx	r13d, BYTE PTR [r12+rcx]
 7147        mov	BYTE PTR [rsp+rbx], r13b
 7148        inc	ecx
 7149        inc	ebx
 7150        cmp	ecx, edx
 7151        jl	L_AES_GCM_encrypt_avx1_calc_aad_loop
 7152        vmovdqu	xmm8, OWORD PTR [rsp]
 7153        add	rsp, 16
 7154        vpshufb	xmm8, xmm8, OWORD PTR L_avx1_aes_gcm_bswap_mask
 7155        vpxor	xmm6, xmm6, xmm8
 7156        ; ghash_gfmul_avx
 7157        vpshufd	xmm1, xmm6, 78
 7158        vpshufd	xmm2, xmm5, 78
 7159        vpclmulqdq	xmm3, xmm5, xmm6, 17
 7160        vpclmulqdq	xmm0, xmm5, xmm6, 0
 7161        vpxor	xmm1, xmm1, xmm6
 7162        vpxor	xmm2, xmm2, xmm5
 7163        vpclmulqdq	xmm1, xmm1, xmm2, 0
 7164        vpxor	xmm1, xmm1, xmm0
 7165        vpxor	xmm1, xmm1, xmm3
 7166        vmovdqa	xmm7, xmm0
 7167        vmovdqa	xmm6, xmm3
 7168        vpslldq	xmm2, xmm1, 8
 7169        vpsrldq	xmm1, xmm1, 8
 7170        vpxor	xmm7, xmm7, xmm2
 7171        vpxor	xmm6, xmm6, xmm1
 7172        vpsrld	xmm0, xmm7, 31
 7173        vpsrld	xmm1, xmm6, 31
 7174        vpslld	xmm7, xmm7, 1
 7175        vpslld	xmm6, xmm6, 1
 7176        vpsrldq	xmm2, xmm0, 12
 7177        vpslldq	xmm0, xmm0, 4
 7178        vpslldq	xmm1, xmm1, 4
 7179        vpor	xmm6, xmm6, xmm2
 7180        vpor	xmm7, xmm7, xmm0
 7181        vpor	xmm6, xmm6, xmm1
 7182        vpslld	xmm0, xmm7, 31
 7183        vpslld	xmm1, xmm7, 30
 7184        vpslld	xmm2, xmm7, 25
 7185        vpxor	xmm0, xmm0, xmm1
 7186        vpxor	xmm0, xmm0, xmm2
 7187        vmovdqa	xmm1, xmm0
 7188        vpsrldq	xmm1, xmm1, 4
 7189        vpslldq	xmm0, xmm0, 12
 7190        vpxor	xmm7, xmm7, xmm0
 7191        vpsrld	xmm2, xmm7, 1
 7192        vpsrld	xmm3, xmm7, 2
 7193        vpsrld	xmm0, xmm7, 7
 7194        vpxor	xmm2, xmm2, xmm3
 7195        vpxor	xmm2, xmm2, xmm0
 7196        vpxor	xmm2, xmm2, xmm1
 7197        vpxor	xmm2, xmm2, xmm7
 7198        vpxor	xmm6, xmm6, xmm2
 7199L_AES_GCM_encrypt_avx1_calc_aad_done:
 7200        ; Calculate counter and H
 7201        vpsrlq	xmm9, xmm5, 63
 7202        vpsllq	xmm8, xmm5, 1
 7203        vpslldq	xmm9, xmm9, 8
 7204        vpor	xmm8, xmm8, xmm9
 7205        vpshufd	xmm5, xmm5, 255
 7206        vpsrad	xmm5, xmm5, 31
 7207        vpshufb	xmm4, xmm4, OWORD PTR L_avx1_aes_gcm_bswap_epi64
 7208        vpand	xmm5, xmm5, OWORD PTR L_avx1_aes_gcm_mod2_128
 7209        vpaddd	xmm4, xmm4, OWORD PTR L_avx1_aes_gcm_one
 7210        vpxor	xmm5, xmm5, xmm8
 7211        vmovdqu	OWORD PTR [rsp+128], xmm4
 7212        xor	ebx, ebx
 7213        cmp	r9d, 128
 7214        mov	r13d, r9d
 7215        jl	L_AES_GCM_encrypt_avx1_done_128
 7216        and	r13d, 4294967168
 7217        vmovdqa	xmm2, xmm6
 7218        ; H ^ 1
 7219        vmovdqu	OWORD PTR [rsp], xmm5
 7220        ; H ^ 2
 7221        vpclmulqdq	xmm8, xmm5, xmm5, 0
 7222        vpclmulqdq	xmm0, xmm5, xmm5, 17
 7223        vpslld	xmm12, xmm8, 31
 7224        vpslld	xmm13, xmm8, 30
 7225        vpslld	xmm14, xmm8, 25
 7226        vpxor	xmm12, xmm12, xmm13
 7227        vpxor	xmm12, xmm12, xmm14
 7228        vpsrldq	xmm13, xmm12, 4
 7229        vpslldq	xmm12, xmm12, 12
 7230        vpxor	xmm8, xmm8, xmm12
 7231        vpsrld	xmm14, xmm8, 1
 7232        vpsrld	xmm10, xmm8, 2
 7233        vpsrld	xmm9, xmm8, 7
 7234        vpxor	xmm14, xmm14, xmm10
 7235        vpxor	xmm14, xmm14, xmm9
 7236        vpxor	xmm14, xmm14, xmm13
 7237        vpxor	xmm14, xmm14, xmm8
 7238        vpxor	xmm0, xmm0, xmm14
 7239        vmovdqu	OWORD PTR [rsp+16], xmm0
 7240        ; H ^ 3
 7241        ; ghash_gfmul_red_avx
 7242        vpshufd	xmm9, xmm5, 78
 7243        vpshufd	xmm10, xmm0, 78
 7244        vpclmulqdq	xmm11, xmm0, xmm5, 17
 7245        vpclmulqdq	xmm8, xmm0, xmm5, 0
 7246        vpxor	xmm9, xmm9, xmm5
 7247        vpxor	xmm10, xmm10, xmm0
 7248        vpclmulqdq	xmm9, xmm9, xmm10, 0
 7249        vpxor	xmm9, xmm9, xmm8
 7250        vpxor	xmm9, xmm9, xmm11
 7251        vpslldq	xmm10, xmm9, 8
 7252        vpsrldq	xmm9, xmm9, 8
 7253        vpxor	xmm8, xmm8, xmm10
 7254        vpxor	xmm1, xmm11, xmm9
 7255        vpslld	xmm12, xmm8, 31
 7256        vpslld	xmm13, xmm8, 30
 7257        vpslld	xmm14, xmm8, 25
 7258        vpxor	xmm12, xmm12, xmm13
 7259        vpxor	xmm12, xmm12, xmm14
 7260        vpsrldq	xmm13, xmm12, 4
 7261        vpslldq	xmm12, xmm12, 12
 7262        vpxor	xmm8, xmm8, xmm12
 7263        vpsrld	xmm14, xmm8, 1
 7264        vpsrld	xmm10, xmm8, 2
 7265        vpsrld	xmm9, xmm8, 7
 7266        vpxor	xmm14, xmm14, xmm10
 7267        vpxor	xmm14, xmm14, xmm9
 7268        vpxor	xmm14, xmm14, xmm13
 7269        vpxor	xmm14, xmm14, xmm8
 7270        vpxor	xmm1, xmm1, xmm14
 7271        vmovdqu	OWORD PTR [rsp+32], xmm1
 7272        ; H ^ 4
 7273        vpclmulqdq	xmm8, xmm0, xmm0, 0
 7274        vpclmulqdq	xmm3, xmm0, xmm0, 17
 7275        vpslld	xmm12, xmm8, 31
 7276        vpslld	xmm13, xmm8, 30
 7277        vpslld	xmm14, xmm8, 25
 7278        vpxor	xmm12, xmm12, xmm13
 7279        vpxor	xmm12, xmm12, xmm14
 7280        vpsrldq	xmm13, xmm12, 4
 7281        vpslldq	xmm12, xmm12, 12
 7282        vpxor	xmm8, xmm8, xmm12
 7283        vpsrld	xmm14, xmm8, 1
 7284        vpsrld	xmm10, xmm8, 2
 7285        vpsrld	xmm9, xmm8, 7
 7286        vpxor	xmm14, xmm14, xmm10
 7287        vpxor	xmm14, xmm14, xmm9
 7288        vpxor	xmm14, xmm14, xmm13
 7289        vpxor	xmm14, xmm14, xmm8
 7290        vpxor	xmm3, xmm3, xmm14
 7291        vmovdqu	OWORD PTR [rsp+48], xmm3
 7292        ; H ^ 5
 7293        ; ghash_gfmul_red_avx
 7294        vpshufd	xmm9, xmm0, 78
 7295        vpshufd	xmm10, xmm1, 78
 7296        vpclmulqdq	xmm11, xmm1, xmm0, 17
 7297        vpclmulqdq	xmm8, xmm1, xmm0, 0
 7298        vpxor	xmm9, xmm9, xmm0
 7299        vpxor	xmm10, xmm10, xmm1
 7300        vpclmulqdq	xmm9, xmm9, xmm10, 0
 7301        vpxor	xmm9, xmm9, xmm8
 7302        vpxor	xmm9, xmm9, xmm11
 7303        vpslldq	xmm10, xmm9, 8
 7304        vpsrldq	xmm9, xmm9, 8
 7305        vpxor	xmm8, xmm8, xmm10
 7306        vpxor	xmm7, xmm11, xmm9
 7307        vpslld	xmm12, xmm8, 31
 7308        vpslld	xmm13, xmm8, 30
 7309        vpslld	xmm14, xmm8, 25
 7310        vpxor	xmm12, xmm12, xmm13
 7311        vpxor	xmm12, xmm12, xmm14
 7312        vpsrldq	xmm13, xmm12, 4
 7313        vpslldq	xmm12, xmm12, 12
 7314        vpxor	xmm8, xmm8, xmm12
 7315        vpsrld	xmm14, xmm8, 1
 7316        vpsrld	xmm10, xmm8, 2
 7317        vpsrld	xmm9, xmm8, 7
 7318        vpxor	xmm14, xmm14, xmm10
 7319        vpxor	xmm14, xmm14, xmm9
 7320        vpxor	xmm14, xmm14, xmm13
 7321        vpxor	xmm14, xmm14, xmm8
 7322        vpxor	xmm7, xmm7, xmm14
 7323        vmovdqu	OWORD PTR [rsp+64], xmm7
 7324        ; H ^ 6
 7325        vpclmulqdq	xmm8, xmm1, xmm1, 0
 7326        vpclmulqdq	xmm7, xmm1, xmm1, 17
 7327        vpslld	xmm12, xmm8, 31
 7328        vpslld	xmm13, xmm8, 30
 7329        vpslld	xmm14, xmm8, 25
 7330        vpxor	xmm12, xmm12, xmm13
 7331        vpxor	xmm12, xmm12, xmm14
 7332        vpsrldq	xmm13, xmm12, 4
 7333        vpslldq	xmm12, xmm12, 12
 7334        vpxor	xmm8, xmm8, xmm12
 7335        vpsrld	xmm14, xmm8, 1
 7336        vpsrld	xmm10, xmm8, 2
 7337        vpsrld	xmm9, xmm8, 7
 7338        vpxor	xmm14, xmm14, xmm10
 7339        vpxor	xmm14, xmm14, xmm9
 7340        vpxor	xmm14, xmm14, xmm13
 7341        vpxor	xmm14, xmm14, xmm8
 7342        vpxor	xmm7, xmm7, xmm14
 7343        vmovdqu	OWORD PTR [rsp+80], xmm7
 7344        ; H ^ 7
 7345        ; ghash_gfmul_red_avx
 7346        vpshufd	xmm9, xmm1, 78
 7347        vpshufd	xmm10, xmm3, 78
 7348        vpclmulqdq	xmm11, xmm3, xmm1, 17
 7349        vpclmulqdq	xmm8, xmm3, xmm1, 0
 7350        vpxor	xmm9, xmm9, xmm1
 7351        vpxor	xmm10, xmm10, xmm3
 7352        vpclmulqdq	xmm9, xmm9, xmm10, 0
 7353        vpxor	xmm9, xmm9, xmm8
 7354        vpxor	xmm9, xmm9, xmm11
 7355        vpslldq	xmm10, xmm9, 8
 7356        vpsrldq	xmm9, xmm9, 8
 7357        vpxor	xmm8, xmm8, xmm10
 7358        vpxor	xmm7, xmm11, xmm9
 7359        vpslld	xmm12, xmm8, 31
 7360        vpslld	xmm13, xmm8, 30
 7361        vpslld	xmm14, xmm8, 25
 7362        vpxor	xmm12, xmm12, xmm13
 7363        vpxor	xmm12, xmm12, xmm14
 7364        vpsrldq	xmm13, xmm12, 4
 7365        vpslldq	xmm12, xmm12, 12
 7366        vpxor	xmm8, xmm8, xmm12
 7367        vpsrld	xmm14, xmm8, 1
 7368        vpsrld	xmm10, xmm8, 2
 7369        vpsrld	xmm9, xmm8, 7
 7370        vpxor	xmm14, xmm14, xmm10
 7371        vpxor	xmm14, xmm14, xmm9
 7372        vpxor	xmm14, xmm14, xmm13
 7373        vpxor	xmm14, xmm14, xmm8
 7374        vpxor	xmm7, xmm7, xmm14
 7375        vmovdqu	OWORD PTR [rsp+96], xmm7
 7376        ; H ^ 8
 7377        vpclmulqdq	xmm8, xmm3, xmm3, 0
 7378        vpclmulqdq	xmm7, xmm3, xmm3, 17
 7379        vpslld	xmm12, xmm8, 31
 7380        vpslld	xmm13, xmm8, 30
 7381        vpslld	xmm14, xmm8, 25
 7382        vpxor	xmm12, xmm12, xmm13
 7383        vpxor	xmm12, xmm12, xmm14
 7384        vpsrldq	xmm13, xmm12, 4
 7385        vpslldq	xmm12, xmm12, 12
 7386        vpxor	xmm8, xmm8, xmm12
 7387        vpsrld	xmm14, xmm8, 1
 7388        vpsrld	xmm10, xmm8, 2
 7389        vpsrld	xmm9, xmm8, 7
 7390        vpxor	xmm14, xmm14, xmm10
 7391        vpxor	xmm14, xmm14, xmm9
 7392        vpxor	xmm14, xmm14, xmm13
 7393        vpxor	xmm14, xmm14, xmm8
 7394        vpxor	xmm7, xmm7, xmm14
 7395        vmovdqu	OWORD PTR [rsp+112], xmm7
 7396        ; First 128 bytes of input
 7397        vmovdqu	xmm0, OWORD PTR [rsp+128]
 7398        vmovdqa	xmm1, OWORD PTR L_avx1_aes_gcm_bswap_epi64
 7399        vpshufb	xmm8, xmm0, xmm1
 7400        vpaddd	xmm9, xmm0, OWORD PTR L_avx1_aes_gcm_one
 7401        vpshufb	xmm9, xmm9, xmm1
 7402        vpaddd	xmm10, xmm0, OWORD PTR L_avx1_aes_gcm_two
 7403        vpshufb	xmm10, xmm10, xmm1
 7404        vpaddd	xmm11, xmm0, OWORD PTR L_avx1_aes_gcm_three
 7405        vpshufb	xmm11, xmm11, xmm1
 7406        vpaddd	xmm12, xmm0, OWORD PTR L_avx1_aes_gcm_four
 7407        vpshufb	xmm12, xmm12, xmm1
 7408        vpaddd	xmm13, xmm0, OWORD PTR L_avx1_aes_gcm_five
 7409        vpshufb	xmm13, xmm13, xmm1
 7410        vpaddd	xmm14, xmm0, OWORD PTR L_avx1_aes_gcm_six
 7411        vpshufb	xmm14, xmm14, xmm1
 7412        vpaddd	xmm15, xmm0, OWORD PTR L_avx1_aes_gcm_seven
 7413        vpshufb	xmm15, xmm15, xmm1
 7414        vpaddd	xmm0, xmm0, OWORD PTR L_avx1_aes_gcm_eight
 7415        vmovdqa	xmm7, OWORD PTR [r15]
 7416        vmovdqu	OWORD PTR [rsp+128], xmm0
 7417        vpxor	xmm8, xmm8, xmm7
 7418        vpxor	xmm9, xmm9, xmm7
 7419        vpxor	xmm10, xmm10, xmm7
 7420        vpxor	xmm11, xmm11, xmm7
 7421        vpxor	xmm12, xmm12, xmm7
 7422        vpxor	xmm13, xmm13, xmm7
 7423        vpxor	xmm14, xmm14, xmm7
 7424        vpxor	xmm15, xmm15, xmm7
 7425        vmovdqa	xmm7, OWORD PTR [r15+16]
 7426        vaesenc	xmm8, xmm8, xmm7
 7427        vaesenc	xmm9, xmm9, xmm7
 7428        vaesenc	xmm10, xmm10, xmm7
 7429        vaesenc	xmm11, xmm11, xmm7
 7430        vaesenc	xmm12, xmm12, xmm7
 7431        vaesenc	xmm13, xmm13, xmm7
 7432        vaesenc	xmm14, xmm14, xmm7
 7433        vaesenc	xmm15, xmm15, xmm7
 7434        vmovdqa	xmm7, OWORD PTR [r15+32]
 7435        vaesenc	xmm8, xmm8, xmm7
 7436        vaesenc	xmm9, xmm9, xmm7
 7437        vaesenc	xmm10, xmm10, xmm7
 7438        vaesenc	xmm11, xmm11, xmm7
 7439        vaesenc	xmm12, xmm12, xmm7
 7440        vaesenc	xmm13, xmm13, xmm7
 7441        vaesenc	xmm14, xmm14, xmm7
 7442        vaesenc	xmm15, xmm15, xmm7
 7443        vmovdqa	xmm7, OWORD PTR [r15+48]
 7444        vaesenc	xmm8, xmm8, xmm7
 7445        vaesenc	xmm9, xmm9, xmm7
 7446        vaesenc	xmm10, xmm10, xmm7
 7447        vaesenc	xmm11, xmm11, xmm7
 7448        vaesenc	xmm12, xmm12, xmm7
 7449        vaesenc	xmm13, xmm13, xmm7
 7450        vaesenc	xmm14, xmm14, xmm7
 7451        vaesenc	xmm15, xmm15, xmm7
 7452        vmovdqa	xmm7, OWORD PTR [r15+64]
 7453        vaesenc	xmm8, xmm8, xmm7
 7454        vaesenc	xmm9, xmm9, xmm7
 7455        vaesenc	xmm10, xmm10, xmm7
 7456        vaesenc	xmm11, xmm11, xmm7
 7457        vaesenc	xmm12, xmm12, xmm7
 7458        vaesenc	xmm13, xmm13, xmm7
 7459        vaesenc	xmm14, xmm14, xmm7
 7460        vaesenc	xmm15, xmm15, xmm7
 7461        vmovdqa	xmm7, OWORD PTR [r15+80]
 7462        vaesenc	xmm8, xmm8, xmm7
 7463        vaesenc	xmm9, xmm9, xmm7
 7464        vaesenc	xmm10, xmm10, xmm7
 7465        vaesenc	xmm11, xmm11, xmm7
 7466        vaesenc	xmm12, xmm12, xmm7
 7467        vaesenc	xmm13, xmm13, xmm7
 7468        vaesenc	xmm14, xmm14, xmm7
 7469        vaesenc	xmm15, xmm15, xmm7
 7470        vmovdqa	xmm7, OWORD PTR [r15+96]
 7471        vaesenc	xmm8, xmm8, xmm7
 7472        vaesenc	xmm9, xmm9, xmm7
 7473        vaesenc	xmm10, xmm10, xmm7
 7474        vaesenc	xmm11, xmm11, xmm7
 7475        vaesenc	xmm12, xmm12, xmm7
 7476        vaesenc	xmm13, xmm13, xmm7
 7477        vaesenc	xmm14, xmm14, xmm7
 7478        vaesenc	xmm15, xmm15, xmm7
 7479        vmovdqa	xmm7, OWORD PTR [r15+112]
 7480        vaesenc	xmm8, xmm8, xmm7
 7481        vaesenc	xmm9, xmm9, xmm7
 7482        vaesenc	xmm10, xmm10, xmm7
 7483        vaesenc	xmm11, xmm11, xmm7
 7484        vaesenc	xmm12, xmm12, xmm7
 7485        vaesenc	xmm13, xmm13, xmm7
 7486        vaesenc	xmm14, xmm14, xmm7
 7487        vaesenc	xmm15, xmm15, xmm7
 7488        vmovdqa	xmm7, OWORD PTR [r15+128]
 7489        vaesenc	xmm8, xmm8, xmm7
 7490        vaesenc	xmm9, xmm9, xmm7
 7491        vaesenc	xmm10, xmm10, xmm7
 7492        vaesenc	xmm11, xmm11, xmm7
 7493        vaesenc	xmm12, xmm12, xmm7
 7494        vaesenc	xmm13, xmm13, xmm7
 7495        vaesenc	xmm14, xmm14, xmm7
 7496        vaesenc	xmm15, xmm15, xmm7
 7497        vmovdqa	xmm7, OWORD PTR [r15+144]
 7498        vaesenc	xmm8, xmm8, xmm7
 7499        vaesenc	xmm9, xmm9, xmm7
 7500        vaesenc	xmm10, xmm10, xmm7
 7501        vaesenc	xmm11, xmm11, xmm7
 7502        vaesenc	xmm12, xmm12, xmm7
 7503        vaesenc	xmm13, xmm13, xmm7
 7504        vaesenc	xmm14, xmm14, xmm7
 7505        vaesenc	xmm15, xmm15, xmm7
 7506        cmp	r10d, 11
 7507        vmovdqa	xmm7, OWORD PTR [r15+160]
 7508        jl	L_AES_GCM_encrypt_avx1_aesenc_128_enc_done
 7509        vaesenc	xmm8, xmm8, xmm7
 7510        vaesenc	xmm9, xmm9, xmm7
 7511        vaesenc	xmm10, xmm10, xmm7
 7512        vaesenc	xmm11, xmm11, xmm7
 7513        vaesenc	xmm12, xmm12, xmm7
 7514        vaesenc	xmm13, xmm13, xmm7
 7515        vaesenc	xmm14, xmm14, xmm7
 7516        vaesenc	xmm15, xmm15, xmm7
 7517        vmovdqa	xmm7, OWORD PTR [r15+176]
 7518        vaesenc	xmm8, xmm8, xmm7
 7519        vaesenc	xmm9, xmm9, xmm7
 7520        vaesenc	xmm10, xmm10, xmm7
 7521        vaesenc	xmm11, xmm11, xmm7
 7522        vaesenc	xmm12, xmm12, xmm7
 7523        vaesenc	xmm13, xmm13, xmm7
 7524        vaesenc	xmm14, xmm14, xmm7
 7525        vaesenc	xmm15, xmm15, xmm7
 7526        cmp	r10d, 13
 7527        vmovdqa	xmm7, OWORD PTR [r15+192]
 7528        jl	L_AES_GCM_encrypt_avx1_aesenc_128_enc_done
 7529        vaesenc	xmm8, xmm8, xmm7
 7530        vaesenc	xmm9, xmm9, xmm7
 7531        vaesenc	xmm10, xmm10, xmm7
 7532        vaesenc	xmm11, xmm11, xmm7
 7533        vaesenc	xmm12, xmm12, xmm7
 7534        vaesenc	xmm13, xmm13, xmm7
 7535        vaesenc	xmm14, xmm14, xmm7
 7536        vaesenc	xmm15, xmm15, xmm7
 7537        vmovdqa	xmm7, OWORD PTR [r15+208]
 7538        vaesenc	xmm8, xmm8, xmm7
 7539        vaesenc	xmm9, xmm9, xmm7
 7540        vaesenc	xmm10, xmm10, xmm7
 7541        vaesenc	xmm11, xmm11, xmm7
 7542        vaesenc	xmm12, xmm12, xmm7
 7543        vaesenc	xmm13, xmm13, xmm7
 7544        vaesenc	xmm14, xmm14, xmm7
 7545        vaesenc	xmm15, xmm15, xmm7
 7546        vmovdqa	xmm7, OWORD PTR [r15+224]
 7547L_AES_GCM_encrypt_avx1_aesenc_128_enc_done:
 7548        vaesenclast	xmm8, xmm8, xmm7
 7549        vaesenclast	xmm9, xmm9, xmm7
 7550        vmovdqu	xmm0, OWORD PTR [rdi]
 7551        vmovdqu	xmm1, OWORD PTR [rdi+16]
 7552        vpxor	xmm8, xmm8, xmm0
 7553        vpxor	xmm9, xmm9, xmm1
 7554        vmovdqu	OWORD PTR [rsi], xmm8
 7555        vmovdqu	OWORD PTR [rsi+16], xmm9
 7556        vaesenclast	xmm10, xmm10, xmm7
 7557        vaesenclast	xmm11, xmm11, xmm7
 7558        vmovdqu	xmm0, OWORD PTR [rdi+32]
 7559        vmovdqu	xmm1, OWORD PTR [rdi+48]
 7560        vpxor	xmm10, xmm10, xmm0
 7561        vpxor	xmm11, xmm11, xmm1
 7562        vmovdqu	OWORD PTR [rsi+32], xmm10
 7563        vmovdqu	OWORD PTR [rsi+48], xmm11
 7564        vaesenclast	xmm12, xmm12, xmm7
 7565        vaesenclast	xmm13, xmm13, xmm7
 7566        vmovdqu	xmm0, OWORD PTR [rdi+64]
 7567        vmovdqu	xmm1, OWORD PTR [rdi+80]
 7568        vpxor	xmm12, xmm12, xmm0
 7569        vpxor	xmm13, xmm13, xmm1
 7570        vmovdqu	OWORD PTR [rsi+64], xmm12
 7571        vmovdqu	OWORD PTR [rsi+80], xmm13
 7572        vaesenclast	xmm14, xmm14, xmm7
 7573        vaesenclast	xmm15, xmm15, xmm7
 7574        vmovdqu	xmm0, OWORD PTR [rdi+96]
 7575        vmovdqu	xmm1, OWORD PTR [rdi+112]
 7576        vpxor	xmm14, xmm14, xmm0
 7577        vpxor	xmm15, xmm15, xmm1
 7578        vmovdqu	OWORD PTR [rsi+96], xmm14
 7579        vmovdqu	OWORD PTR [rsi+112], xmm15
 7580        cmp	r13d, 128
 7581        mov	ebx, 128
 7582        jle	L_AES_GCM_encrypt_avx1_end_128
 7583        ; More 128 bytes of input
 7584L_AES_GCM_encrypt_avx1_ghash_128:
 7585        lea	rcx, QWORD PTR [rdi+rbx]
 7586        lea	rdx, QWORD PTR [rsi+rbx]
 7587        vmovdqu	xmm0, OWORD PTR [rsp+128]
 7588        vmovdqa	xmm1, OWORD PTR L_avx1_aes_gcm_bswap_epi64
 7589        vpshufb	xmm8, xmm0, xmm1
 7590        vpaddd	xmm9, xmm0, OWORD PTR L_avx1_aes_gcm_one
 7591        vpshufb	xmm9, xmm9, xmm1
 7592        vpaddd	xmm10, xmm0, OWORD PTR L_avx1_aes_gcm_two
 7593        vpshufb	xmm10, xmm10, xmm1
 7594        vpaddd	xmm11, xmm0, OWORD PTR L_avx1_aes_gcm_three
 7595        vpshufb	xmm11, xmm11, xmm1
 7596        vpaddd	xmm12, xmm0, OWORD PTR L_avx1_aes_gcm_four
 7597        vpshufb	xmm12, xmm12, xmm1
 7598        vpaddd	xmm13, xmm0, OWORD PTR L_avx1_aes_gcm_five
 7599        vpshufb	xmm13, xmm13, xmm1
 7600        vpaddd	xmm14, xmm0, OWORD PTR L_avx1_aes_gcm_six
 7601        vpshufb	xmm14, xmm14, xmm1
 7602        vpaddd	xmm15, xmm0, OWORD PTR L_avx1_aes_gcm_seven
 7603        vpshufb	xmm15, xmm15, xmm1
 7604        vpaddd	xmm0, xmm0, OWORD PTR L_avx1_aes_gcm_eight
 7605        vmovdqa	xmm7, OWORD PTR [r15]
 7606        vmovdqu	OWORD PTR [rsp+128], xmm0
 7607        vpxor	xmm8, xmm8, xmm7
 7608        vpxor	xmm9, xmm9, xmm7
 7609        vpxor	xmm10, xmm10, xmm7
 7610        vpxor	xmm11, xmm11, xmm7
 7611        vpxor	xmm12, xmm12, xmm7
 7612        vpxor	xmm13, xmm13, xmm7
 7613        vpxor	xmm14, xmm14, xmm7
 7614        vpxor	xmm15, xmm15, xmm7
 7615        vmovdqu	xmm7, OWORD PTR [rsp+112]
 7616        vmovdqu	xmm0, OWORD PTR [rdx+-128]
 7617        vaesenc	xmm8, xmm8, [r15+16]
 7618        vpshufb	xmm0, xmm0, OWORD PTR L_avx1_aes_gcm_bswap_mask
 7619        vpxor	xmm0, xmm0, xmm2
 7620        vpshufd	xmm1, xmm7, 78
 7621        vpshufd	xmm5, xmm0, 78
 7622        vpxor	xmm1, xmm1, xmm7
 7623        vpxor	xmm5, xmm5, xmm0
 7624        vpclmulqdq	xmm3, xmm0, xmm7, 17
 7625        vaesenc	xmm9, xmm9, [r15+16]
 7626        vaesenc	xmm10, xmm10, [r15+16]
 7627        vpclmulqdq	xmm2, xmm0, xmm7, 0
 7628        vaesenc	xmm11, xmm11, [r15+16]
 7629        vaesenc	xmm12, xmm12, [r15+16]
 7630        vpclmulqdq	xmm1, xmm1, xmm5, 0
 7631        vaesenc	xmm13, xmm13, [r15+16]
 7632        vaesenc	xmm14, xmm14, [r15+16]
 7633        vaesenc	xmm15, xmm15, [r15+16]
 7634        vpxor	xmm1, xmm1, xmm2
 7635        vpxor	xmm1, xmm1, xmm3
 7636        vmovdqu	xmm7, OWORD PTR [rsp+96]
 7637        vmovdqu	xmm0, OWORD PTR [rdx+-112]
 7638        vpshufd	xmm4, xmm7, 78
 7639        vpshufb	xmm0, xmm0, OWORD PTR L_avx1_aes_gcm_bswap_mask
 7640        vaesenc	xmm8, xmm8, [r15+32]
 7641        vpxor	xmm4, xmm4, xmm7
 7642        vpshufd	xmm5, xmm0, 78
 7643        vpxor	xmm5, xmm5, xmm0
 7644        vpclmulqdq	xmm6, xmm0, xmm7, 17
 7645        vaesenc	xmm9, xmm9, [r15+32]
 7646        vaesenc	xmm10, xmm10, [r15+32]
 7647        vpclmulqdq	xmm7, xmm0, xmm7, 0
 7648        vaesenc	xmm11, xmm11, [r15+32]
 7649        vaesenc	xmm12, xmm12, [r15+32]
 7650        vpclmulqdq	xmm4, xmm4, xmm5, 0
 7651        vaesenc	xmm13, xmm13, [r15+32]
 7652        vaesenc	xmm14, xmm14, [r15+32]
 7653        vaesenc	xmm15, xmm15, [r15+32]
 7654        vpxor	xmm1, xmm1, xmm7
 7655        vpxor	xmm2, xmm2, xmm7
 7656        vpxor	xmm1, xmm1, xmm6
 7657        vpxor	xmm3, xmm3, xmm6
 7658        vpxor	xmm1, xmm1, xmm4
 7659        vmovdqu	xmm7, OWORD PTR [rsp+80]
 7660        vmovdqu	xmm0, OWORD PTR [rdx+-96]
 7661        vpshufd	xmm4, xmm7, 78
 7662        vpshufb	xmm0, xmm0, OWORD PTR L_avx1_aes_gcm_bswap_mask
 7663        vaesenc	xmm8, xmm8, [r15+48]
 7664        vpxor	xmm4, xmm4, xmm7
 7665        vpshufd	xmm5, xmm0, 78
 7666        vpxor	xmm5, xmm5, xmm0
 7667        vpclmulqdq	xmm6, xmm0, xmm7, 17
 7668        vaesenc	xmm9, xmm9, [r15+48]
 7669        vaesenc	xmm10, xmm10, [r15+48]
 7670        vpclmulqdq	xmm7, xmm0, xmm7, 0
 7671        vaesenc	xmm11, xmm11, [r15+48]
 7672        vaesenc	xmm12, xmm12, [r15+48]
 7673        vpclmulqdq	xmm4, xmm4, xmm5, 0
 7674        vaesenc	xmm13, xmm13, [r15+48]
 7675        vaesenc	xmm14, xmm14, [r15+48]
 7676        vaesenc	xmm15, xmm15, [r15+48]
 7677        vpxor	xmm1, xmm1, xmm7
 7678        vpxor	xmm2, xmm2, xmm7
 7679        vpxor	xmm1, xmm1, xmm6
 7680        vpxor	xmm3, xmm3, xmm6
 7681        vpxor	xmm1, xmm1, xmm4
 7682        vmovdqu	xmm7, OWORD PTR [rsp+64]
 7683        vmovdqu	xmm0, OWORD PTR [rdx+-80]
 7684        vpshufd	xmm4, xmm7, 78
 7685        vpshufb	xmm0, xmm0, OWORD PTR L_avx1_aes_gcm_bswap_mask
 7686        vaesenc	xmm8, xmm8, [r15+64]
 7687        vpxor	xmm4, xmm4, xmm7
 7688        vpshufd	xmm5, xmm0, 78
 7689        vpxor	xmm5, xmm5, xmm0
 7690        vpclmulqdq	xmm6, xmm0, xmm7, 17
 7691        vaesenc	xmm9, xmm9, [r15+64]
 7692        vaesenc	xmm10, xmm10, [r15+64]
 7693        vpclmulqdq	xmm7, xmm0, xmm7, 0
 7694        vaesenc	xmm11, xmm11, [r15+64]
 7695        vaesenc	xmm12, xmm12, [r15+64]
 7696        vpclmulqdq	xmm4, xmm4, xmm5, 0
 7697        vaesenc	xmm13, xmm13, [r15+64]
 7698        vaesenc	xmm14, xmm14, [r15+64]
 7699        vaesenc	xmm15, xmm15, [r15+64]
 7700        vpxor	xmm1, xmm1, xmm7
 7701        vpxor	xmm2, xmm2, xmm7
 7702        vpxor	xmm1, xmm1, xmm6
 7703        vpxor	xmm3, xmm3, xmm6
 7704        vpxor	xmm1, xmm1, xmm4
 7705        vmovdqu	xmm7, OWORD PTR [rsp+48]
 7706        vmovdqu	xmm0, OWORD PTR [rdx+-64]
 7707        vpshufd	xmm4, xmm7, 78
 7708        vpshufb	xmm0, xmm0, OWORD PTR L_avx1_aes_gcm_bswap_mask
 7709        vaesenc	xmm8, xmm8, [r15+80]
 7710        vpxor	xmm4, xmm4, xmm7
 7711        vpshufd	xmm5, xmm0, 78
 7712        vpxor	xmm5, xmm5, xmm0
 7713        vpclmulqdq	xmm6, xmm0, xmm7, 17
 7714        vaesenc	xmm9, xmm9, [r15+80]
 7715        vaesenc	xmm10, xmm10, [r15+80]
 7716        vpclmulqdq	xmm7, xmm0, xmm7, 0
 7717        vaesenc	xmm11, xmm11, [r15+80]
 7718        vaesenc	xmm12, xmm12, [r15+80]
 7719        vpclmulqdq	xmm4, xmm4, xmm5, 0
 7720        vaesenc	xmm13, xmm13, [r15+80]
 7721        vaesenc	xmm14, xmm14, [r15+80]
 7722        vaesenc	xmm15, xmm15, [r15+80]
 7723        vpxor	xmm1, xmm1, xmm7
 7724        vpxor	xmm2, xmm2, xmm7
 7725        vpxor	xmm1, xmm1, xmm6
 7726        vpxor	xmm3, xmm3, xmm6
 7727        vpxor	xmm1, xmm1, xmm4
 7728        vmovdqu	xmm7, OWORD PTR [rsp+32]
 7729        vmovdqu	xmm0, OWORD PTR [rdx+-48]
 7730        vpshufd	xmm4, xmm7, 78
 7731        vpshufb	xmm0, xmm0, OWORD PTR L_avx1_aes_gcm_bswap_mask
 7732        vaesenc	xmm8, xmm8, [r15+96]
 7733        vpxor	xmm4, xmm4, xmm7
 7734        vpshufd	xmm5, xmm0, 78
 7735        vpxor	xmm5, xmm5, xmm0
 7736        vpclmulqdq	xmm6, xmm0, xmm7, 17
 7737        vaesenc	xmm9, xmm9, [r15+96]
 7738        vaesenc	xmm10, xmm10, [r15+96]
 7739        vpclmulqdq	xmm7, xmm0, xmm7, 0
 7740        vaesenc	xmm11, xmm11, [r15+96]
 7741        vaesenc	xmm12, xmm12, [r15+96]
 7742        vpclmulqdq	xmm4, xmm4, xmm5, 0
 7743        vaesenc	xmm13, xmm13, [r15+96]
 7744        vaesenc	xmm14, xmm14, [r15+96]
 7745        vaesenc	xmm15, xmm15, [r15+96]
 7746        vpxor	xmm1, xmm1, xmm7
 7747        vpxor	xmm2, xmm2, xmm7
 7748        vpxor	xmm1, xmm1, xmm6
 7749        vpxor	xmm3, xmm3, xmm6
 7750        vpxor	xmm1, xmm1, xmm4
 7751        vmovdqu	xmm7, OWORD PTR [rsp+16]
 7752        vmovdqu	xmm0, OWORD PTR [rdx+-32]
 7753        vpshufd	xmm4, xmm7, 78
 7754        vpshufb	xmm0, xmm0, OWORD PTR L_avx1_aes_gcm_bswap_mask
 7755        vaesenc	xmm8, xmm8, [r15+112]
 7756        vpxor	xmm4, xmm4, xmm7
 7757        vpshufd	xmm5, xmm0, 78
 7758        vpxor	xmm5, xmm5, xmm0
 7759        vpclmulqdq	xmm6, xmm0, xmm7, 17
 7760        vaesenc	xmm9, xmm9, [r15+112]
 7761        vaesenc	xmm10, xmm10, [r15+112]
 7762        vpclmulqdq	xmm7, xmm0, xmm7, 0
 7763        vaesenc	xmm11, xmm11, [r15+112]
 7764        vaesenc	xmm12, xmm12, [r15+112]
 7765        vpclmulqdq	xmm4, xmm4, xmm5, 0
 7766        vaesenc	xmm13, xmm13, [r15+112]
 7767        vaesenc	xmm14, xmm14, [r15+112]
 7768        vaesenc	xmm15, xmm15, [r15+112]
 7769        vpxor	xmm1, xmm1, xmm7
 7770        vpxor	xmm2, xmm2, xmm7
 7771        vpxor	xmm1, xmm1, xmm6
 7772        vpxor	xmm3, xmm3, xmm6
 7773        vpxor	xmm1, xmm1, xmm4
 7774        vmovdqu	xmm7, OWORD PTR [rsp]
 7775        vmovdqu	xmm0, OWORD PTR [rdx+-16]
 7776        vpshufd	xmm4, xmm7, 78
 7777        vpshufb	xmm0, xmm0, OWORD PTR L_avx1_aes_gcm_bswap_mask
 7778        vaesenc	xmm8, xmm8, [r15+128]
 7779        vpxor	xmm4, xmm4, xmm7
 7780        vpshufd	xmm5, xmm0, 78
 7781        vpxor	xmm5, xmm5, xmm0
 7782        vpclmulqdq	xmm6, xmm0, xmm7, 17
 7783        vaesenc	xmm9, xmm9, [r15+128]
 7784        vaesenc	xmm10, xmm10, [r15+128]
 7785        vpclmulqdq	xmm7, xmm0, xmm7, 0
 7786        vaesenc	xmm11, xmm11, [r15+128]
 7787        vaesenc	xmm12, xmm12, [r15+128]
 7788        vpclmulqdq	xmm4, xmm4, xmm5, 0
 7789        vaesenc	xmm13, xmm13, [r15+128]
 7790        vaesenc	xmm14, xmm14, [r15+128]
 7791        vaesenc	xmm15, xmm15, [r15+128]
 7792        vpxor	xmm1, xmm1, xmm7
 7793        vpxor	xmm2, xmm2, xmm7
 7794        vpxor	xmm1, xmm1, xmm6
 7795        vpxor	xmm3, xmm3, xmm6
 7796        vpxor	xmm1, xmm1, xmm4
 7797        vpslldq	xmm5, xmm1, 8
 7798        vpsrldq	xmm1, xmm1, 8
 7799        vaesenc	xmm8, xmm8, [r15+144]
 7800        vpxor	xmm2, xmm2, xmm5
 7801        vpxor	xmm3, xmm3, xmm1
 7802        vaesenc	xmm9, xmm9, [r15+144]
 7803        vpslld	xmm7, xmm2, 31
 7804        vpslld	xmm4, xmm2, 30
 7805        vpslld	xmm5, xmm2, 25
 7806        vaesenc	xmm10, xmm10, [r15+144]
 7807        vpxor	xmm7, xmm7, xmm4
 7808        vpxor	xmm7, xmm7, xmm5
 7809        vaesenc	xmm11, xmm11, [r15+144]
 7810        vpsrldq	xmm4, xmm7, 4
 7811        vpslldq	xmm7, xmm7, 12
 7812        vaesenc	xmm12, xmm12, [r15+144]
 7813        vpxor	xmm2, xmm2, xmm7
 7814        vpsrld	xmm5, xmm2, 1
 7815        vaesenc	xmm13, xmm13, [r15+144]
 7816        vpsrld	xmm1, xmm2, 2
 7817        vpsrld	xmm0, xmm2, 7
 7818        vaesenc	xmm14, xmm14, [r15+144]
 7819        vpxor	xmm5, xmm5, xmm1
 7820        vpxor	xmm5, xmm5, xmm0
 7821        vaesenc	xmm15, xmm15, [r15+144]
 7822        vpxor	xmm5, xmm5, xmm4
 7823        vpxor	xmm2, xmm2, xmm5
 7824        vpxor	xmm2, xmm2, xmm3
 7825        cmp	r10d, 11
 7826        vmovdqa	xmm7, OWORD PTR [r15+160]
 7827        jl	L_AES_GCM_encrypt_avx1_aesenc_128_ghash_avx_done
 7828        vaesenc	xmm8, xmm8, xmm7
 7829        vaesenc	xmm9, xmm9, xmm7
 7830        vaesenc	xmm10, xmm10, xmm7
 7831        vaesenc	xmm11, xmm11, xmm7
 7832        vaesenc	xmm12, xmm12, xmm7
 7833        vaesenc	xmm13, xmm13, xmm7
 7834        vaesenc	xmm14, xmm14, xmm7
 7835        vaesenc	xmm15, xmm15, xmm7
 7836        vmovdqa	xmm7, OWORD PTR [r15+176]
 7837        vaesenc	xmm8, xmm8, xmm7
 7838        vaesenc	xmm9, xmm9, xmm7
 7839        vaesenc	xmm10, xmm10, xmm7
 7840        vaesenc	xmm11, xmm11, xmm7
 7841        vaesenc	xmm12, xmm12, xmm7
 7842        vaesenc	xmm13, xmm13, xmm7
 7843        vaesenc	xmm14, xmm14, xmm7
 7844        vaesenc	xmm15, xmm15, xmm7
 7845        cmp	r10d, 13
 7846        vmovdqa	xmm7, OWORD PTR [r15+192]
 7847        jl	L_AES_GCM_encrypt_avx1_aesenc_128_ghash_avx_done
 7848        vaesenc	xmm8, xmm8, xmm7
 7849        vaesenc	xmm9, xmm9, xmm7
 7850        vaesenc	xmm10, xmm10, xmm7
 7851        vaesenc	xmm11, xmm11, xmm7
 7852        vaesenc	xmm12, xmm12, xmm7
 7853        vaesenc	xmm13, xmm13, xmm7
 7854        vaesenc	xmm14, xmm14, xmm7
 7855        vaesenc	xmm15, xmm15, xmm7
 7856        vmovdqa	xmm7, OWORD PTR [r15+208]
 7857        vaesenc	xmm8, xmm8, xmm7
 7858        vaesenc	xmm9, xmm9, xmm7
 7859        vaesenc	xmm10, xmm10, xmm7
 7860        vaesenc	xmm11, xmm11, xmm7
 7861        vaesenc	xmm12, xmm12, xmm7
 7862        vaesenc	xmm13, xmm13, xmm7
 7863        vaesenc	xmm14, xmm14, xmm7
 7864        vaesenc	xmm15, xmm15, xmm7
 7865        vmovdqa	xmm7, OWORD PTR [r15+224]
 7866L_AES_GCM_encrypt_avx1_aesenc_128_ghash_avx_done:
 7867        vaesenclast	xmm8, xmm8, xmm7
 7868        vaesenclast	xmm9, xmm9, xmm7
 7869        vmovdqu	xmm0, OWORD PTR [rcx]
 7870        vmovdqu	xmm1, OWORD PTR [rcx+16]
 7871        vpxor	xmm8, xmm8, xmm0
 7872        vpxor	xmm9, xmm9, xmm1
 7873        vmovdqu	OWORD PTR [rdx], xmm8
 7874        vmovdqu	OWORD PTR [rdx+16], xmm9
 7875        vaesenclast	xmm10, xmm10, xmm7
 7876        vaesenclast	xmm11, xmm11, xmm7
 7877        vmovdqu	xmm0, OWORD PTR [rcx+32]
 7878        vmovdqu	xmm1, OWORD PTR [rcx+48]
 7879        vpxor	xmm10, xmm10, xmm0
 7880        vpxor	xmm11, xmm11, xmm1
 7881        vmovdqu	OWORD PTR [rdx+32], xmm10
 7882        vmovdqu	OWORD PTR [rdx+48], xmm11
 7883        vaesenclast	xmm12, xmm12, xmm7
 7884        vaesenclast	xmm13, xmm13, xmm7
 7885        vmovdqu	xmm0, OWORD PTR [rcx+64]
 7886        vmovdqu	xmm1, OWORD PTR [rcx+80]
 7887        vpxor	xmm12, xmm12, xmm0
 7888        vpxor	xmm13, xmm13, xmm1
 7889        vmovdqu	OWORD PTR [rdx+64], xmm12
 7890        vmovdqu	OWORD PTR [rdx+80], xmm13
 7891        vaesenclast	xmm14, xmm14, xmm7
 7892        vaesenclast	xmm15, xmm15, xmm7
 7893        vmovdqu	xmm0, OWORD PTR [rcx+96]
 7894        vmovdqu	xmm1, OWORD PTR [rcx+112]
 7895        vpxor	xmm14, xmm14, xmm0
 7896        vpxor	xmm15, xmm15, xmm1
 7897        vmovdqu	OWORD PTR [rdx+96], xmm14
 7898        vmovdqu	OWORD PTR [rdx+112], xmm15
 7899        add	ebx, 128
 7900        cmp	ebx, r13d
 7901        jl	L_AES_GCM_encrypt_avx1_ghash_128
 7902L_AES_GCM_encrypt_avx1_end_128:
 7903        vmovdqa	xmm4, OWORD PTR L_avx1_aes_gcm_bswap_mask
 7904        vpshufb	xmm8, xmm8, xmm4
 7905        vpshufb	xmm9, xmm9, xmm4
 7906        vpshufb	xmm10, xmm10, xmm4
 7907        vpshufb	xmm11, xmm11, xmm4
 7908        vpxor	xmm8, xmm8, xmm2
 7909        vpshufb	xmm12, xmm12, xmm4
 7910        vpshufb	xmm13, xmm13, xmm4
 7911        vpshufb	xmm14, xmm14, xmm4
 7912        vpshufb	xmm15, xmm15, xmm4
 7913        vmovdqu	xmm7, OWORD PTR [rsp]
 7914        vmovdqu	xmm5, OWORD PTR [rsp+16]
 7915        ; ghash_gfmul_avx
 7916        vpshufd	xmm1, xmm15, 78
 7917        vpshufd	xmm2, xmm7, 78
 7918        vpclmulqdq	xmm3, xmm7, xmm15, 17
 7919        vpclmulqdq	xmm0, xmm7, xmm15, 0
 7920        vpxor	xmm1, xmm1, xmm15
 7921        vpxor	xmm2, xmm2, xmm7
 7922        vpclmulqdq	xmm1, xmm1, xmm2, 0
 7923        vpxor	xmm1, xmm1, xmm0
 7924        vpxor	xmm1, xmm1, xmm3
 7925        vmovdqa	xmm4, xmm0
 7926        vmovdqa	xmm6, xmm3
 7927        vpslldq	xmm2, xmm1, 8
 7928        vpsrldq	xmm1, xmm1, 8
 7929        vpxor	xmm4, xmm4, xmm2
 7930        vpxor	xmm6, xmm6, xmm1
 7931        ; ghash_gfmul_xor_avx
 7932        vpshufd	xmm1, xmm14, 78
 7933        vpshufd	xmm2, xmm5, 78
 7934        vpclmulqdq	xmm3, xmm5, xmm14, 17
 7935        vpclmulqdq	xmm0, xmm5, xmm14, 0
 7936        vpxor	xmm1, xmm1, xmm14
 7937        vpxor	xmm2, xmm2, xmm5
 7938        vpclmulqdq	xmm1, xmm1, xmm2, 0
 7939        vpxor	xmm1, xmm1, xmm0
 7940        vpxor	xmm1, xmm1, xmm3
 7941        vpxor	xmm4, xmm4, xmm0
 7942        vpxor	xmm6, xmm6, xmm3
 7943        vpslldq	xmm2, xmm1, 8
 7944        vpsrldq	xmm1, xmm1, 8
 7945        vpxor	xmm4, xmm4, xmm2
 7946        vpxor	xmm6, xmm6, xmm1
 7947        vmovdqu	xmm7, OWORD PTR [rsp+32]
 7948        vmovdqu	xmm5, OWORD PTR [rsp+48]
 7949        ; ghash_gfmul_xor_avx
 7950        vpshufd	xmm1, xmm13, 78
 7951        vpshufd	xmm2, xmm7, 78
 7952        vpclmulqdq	xmm3, xmm7, xmm13, 17
 7953        vpclmulqdq	xmm0, xmm7, xmm13, 0
 7954        vpxor	xmm1, xmm1, xmm13
 7955        vpxor	xmm2, xmm2, xmm7
 7956        vpclmulqdq	xmm1, xmm1, xmm2, 0
 7957        vpxor	xmm1, xmm1, xmm0
 7958        vpxor	xmm1, xmm1, xmm3
 7959        vpxor	xmm4, xmm4, xmm0
 7960        vpxor	xmm6, xmm6, xmm3
 7961        vpslldq	xmm2, xmm1, 8
 7962        vpsrldq	xmm1, xmm1, 8
 7963        vpxor	xmm4, xmm4, xmm2
 7964        vpxor	xmm6, xmm6, xmm1
 7965        ; ghash_gfmul_xor_avx
 7966        vpshufd	xmm1, xmm12, 78
 7967        vpshufd	xmm2, xmm5, 78
 7968        vpclmulqdq	xmm3, xmm5, xmm12, 17
 7969        vpclmulqdq	xmm0, xmm5, xmm12, 0
 7970        vpxor	xmm1, xmm1, xmm12
 7971        vpxor	xmm2, xmm2, xmm5
 7972        vpclmulqdq	xmm1, xmm1, xmm2, 0
 7973        vpxor	xmm1, xmm1, xmm0
 7974        vpxor	xmm1, xmm1, xmm3
 7975        vpxor	xmm4, xmm4, xmm0
 7976        vpxor	xmm6, xmm6, xmm3
 7977        vpslldq	xmm2, xmm1, 8
 7978        vpsrldq	xmm1, xmm1, 8
 7979        vpxor	xmm4, xmm4, xmm2
 7980        vpxor	xmm6, xmm6, xmm1
 7981        vmovdqu	xmm7, OWORD PTR [rsp+64]
 7982        vmovdqu	xmm5, OWORD PTR [rsp+80]
 7983        ; ghash_gfmul_xor_avx
 7984        vpshufd	xmm1, xmm11, 78
 7985        vpshufd	xmm2, xmm7, 78
 7986        vpclmulqdq	xmm3, xmm7, xmm11, 17
 7987        vpclmulqdq	xmm0, xmm7, xmm11, 0
 7988        vpxor	xmm1, xmm1, xmm11
 7989        vpxor	xmm2, xmm2, xmm7
 7990        vpclmulqdq	xmm1, xmm1, xmm2, 0
 7991        vpxor	xmm1, xmm1, xmm0
 7992        vpxor	xmm1, xmm1, xmm3
 7993        vpxor	xmm4, xmm4, xmm0
 7994        vpxor	xmm6, xmm6, xmm3
 7995        vpslldq	xmm2, xmm1, 8
 7996        vpsrldq	xmm1, xmm1, 8
 7997        vpxor	xmm4, xmm4, xmm2
 7998        vpxor	xmm6, xmm6, xmm1
 7999        ; ghash_gfmul_xor_avx
 8000        vpshufd	xmm1, xmm10, 78
 8001        vpshufd	xmm2, xmm5, 78
 8002        vpclmulqdq	xmm3, xmm5, xmm10, 17
 8003        vpclmulqdq	xmm0, xmm5, xmm10, 0
 8004        vpxor	xmm1, xmm1, xmm10
 8005        vpxor	xmm2, xmm2, xmm5
 8006        vpclmulqdq	xmm1, xmm1, xmm2, 0
 8007        vpxor	xmm1, xmm1, xmm0
 8008        vpxor	xmm1, xmm1, xmm3
 8009        vpxor	xmm4, xmm4, xmm0
 8010        vpxor	xmm6, xmm6, xmm3
 8011        vpslldq	xmm2, xmm1, 8
 8012        vpsrldq	xmm1, xmm1, 8
 8013        vpxor	xmm4, xmm4, xmm2
 8014        vpxor	xmm6, xmm6, xmm1
 8015        vmovdqu	xmm7, OWORD PTR [rsp+96]
 8016        vmovdqu	xmm5, OWORD PTR [rsp+112]
 8017        ; ghash_gfmul_xor_avx
 8018        vpshufd	xmm1, xmm9, 78
 8019        vpshufd	xmm2, xmm7, 78
 8020        vpclmulqdq	xmm3, xmm7, xmm9, 17
 8021        vpclmulqdq	xmm0, xmm7, xmm9, 0
 8022        vpxor	xmm1, xmm1, xmm9
 8023        vpxor	xmm2, xmm2, xmm7
 8024        vpclmulqdq	xmm1, xmm1, xmm2, 0
 8025        vpxor	xmm1, xmm1, xmm0
 8026        vpxor	xmm1, xmm1, xmm3
 8027        vpxor	xmm4, xmm4, xmm0
 8028        vpxor	xmm6, xmm6, xmm3
 8029        vpslldq	xmm2, xmm1, 8
 8030        vpsrldq	xmm1, xmm1, 8
 8031        vpxor	xmm4, xmm4, xmm2
 8032        vpxor	xmm6, xmm6, xmm1
 8033        ; ghash_gfmul_xor_avx
 8034        vpshufd	xmm1, xmm8, 78
 8035        vpshufd	xmm2, xmm5, 78
 8036        vpclmulqdq	xmm3, xmm5, xmm8, 17
 8037        vpclmulqdq	xmm0, xmm5, xmm8, 0
 8038        vpxor	xmm1, xmm1, xmm8
 8039        vpxor	xmm2, xmm2, xmm5
 8040        vpclmulqdq	xmm1, xmm1, xmm2, 0
 8041        vpxor	xmm1, xmm1, xmm0
 8042        vpxor	xmm1, xmm1, xmm3
 8043        vpxor	xmm4, xmm4, xmm0
 8044        vpxor	xmm6, xmm6, xmm3
 8045        vpslldq	xmm2, xmm1, 8
 8046        vpsrldq	xmm1, xmm1, 8
 8047        vpxor	xmm4, xmm4, xmm2
 8048        vpxor	xmm6, xmm6, xmm1
 8049        vpslld	xmm0, xmm4, 31
 8050        vpslld	xmm1, xmm4, 30
 8051        vpslld	xmm2, xmm4, 25
 8052        vpxor	xmm0, xmm0, xmm1
 8053        vpxor	xmm0, xmm0, xmm2
 8054        vmovdqa	xmm1, xmm0
 8055        vpsrldq	xmm1, xmm1, 4
 8056        vpslldq	xmm0, xmm0, 12
 8057        vpxor	xmm4, xmm4, xmm0
 8058        vpsrld	xmm2, xmm4, 1
 8059        vpsrld	xmm3, xmm4, 2
 8060        vpsrld	xmm0, xmm4, 7
 8061        vpxor	xmm2, xmm2, xmm3
 8062        vpxor	xmm2, xmm2, xmm0
 8063        vpxor	xmm2, xmm2, xmm1
 8064        vpxor	xmm2, xmm2, xmm4
 8065        vpxor	xmm6, xmm6, xmm2
 8066        vmovdqu	xmm5, OWORD PTR [rsp]
 8067L_AES_GCM_encrypt_avx1_done_128:
 8068        mov	edx, r9d
 8069        cmp	ebx, edx
 8070        jge	L_AES_GCM_encrypt_avx1_done_enc
 8071        mov	r13d, r9d
 8072        and	r13d, 4294967280
 8073        cmp	ebx, r13d
 8074        jge	L_AES_GCM_encrypt_avx1_last_block_done
 8075        vmovdqu	xmm9, OWORD PTR [rsp+128]
 8076        vpshufb	xmm8, xmm9, OWORD PTR L_avx1_aes_gcm_bswap_epi64
 8077        vpaddd	xmm9, xmm9, OWORD PTR L_avx1_aes_gcm_one
 8078        vmovdqu	OWORD PTR [rsp+128], xmm9
 8079        vpxor	xmm8, xmm8, [r15]
 8080        vaesenc	xmm8, xmm8, [r15+16]
 8081        vaesenc	xmm8, xmm8, [r15+32]
 8082        vaesenc	xmm8, xmm8, [r15+48]
 8083        vaesenc	xmm8, xmm8, [r15+64]
 8084        vaesenc	xmm8, xmm8, [r15+80]
 8085        vaesenc	xmm8, xmm8, [r15+96]
 8086        vaesenc	xmm8, xmm8, [r15+112]
 8087        vaesenc	xmm8, xmm8, [r15+128]
 8088        vaesenc	xmm8, xmm8, [r15+144]
 8089        cmp	r10d, 11
 8090        vmovdqa	xmm9, OWORD PTR [r15+160]
 8091        jl	L_AES_GCM_encrypt_avx1_aesenc_block_last
 8092        vaesenc	xmm8, xmm8, xmm9
 8093        vaesenc	xmm8, xmm8, [r15+176]
 8094        cmp	r10d, 13
 8095        vmovdqa	xmm9, OWORD PTR [r15+192]
 8096        jl	L_AES_GCM_encrypt_avx1_aesenc_block_last
 8097        vaesenc	xmm8, xmm8, xmm9
 8098        vaesenc	xmm8, xmm8, [r15+208]
 8099        vmovdqa	xmm9, OWORD PTR [r15+224]
 8100L_AES_GCM_encrypt_avx1_aesenc_block_last:
 8101        vaesenclast	xmm8, xmm8, xmm9
 8102        vmovdqu	xmm9, OWORD PTR [rdi+rbx]
 8103        vpxor	xmm8, xmm8, xmm9
 8104        vmovdqu	OWORD PTR [rsi+rbx], xmm8
 8105        vpshufb	xmm8, xmm8, OWORD PTR L_avx1_aes_gcm_bswap_mask
 8106        vpxor	xmm6, xmm6, xmm8
 8107        add	ebx, 16
 8108        cmp	ebx, r13d
 8109        jge	L_AES_GCM_encrypt_avx1_last_block_ghash
 8110L_AES_GCM_encrypt_avx1_last_block_start:
 8111        vmovdqu	xmm13, OWORD PTR [rdi+rbx]
 8112        vmovdqu	xmm9, OWORD PTR [rsp+128]
 8113        vpshufb	xmm8, xmm9, OWORD PTR L_avx1_aes_gcm_bswap_epi64
 8114        vpaddd	xmm9, xmm9, OWORD PTR L_avx1_aes_gcm_one
 8115        vmovdqu	OWORD PTR [rsp+128], xmm9
 8116        vpxor	xmm8, xmm8, [r15]
 8117        vpclmulqdq	xmm10, xmm6, xmm5, 16
 8118        vaesenc	xmm8, xmm8, [r15+16]
 8119        vaesenc	xmm8, xmm8, [r15+32]
 8120        vpclmulqdq	xmm11, xmm6, xmm5, 1
 8121        vaesenc	xmm8, xmm8, [r15+48]
 8122        vaesenc	xmm8, xmm8, [r15+64]
 8123        vpclmulqdq	xmm12, xmm6, xmm5, 0
 8124        vaesenc	xmm8, xmm8, [r15+80]
 8125        vpclmulqdq	xmm1, xmm6, xmm5, 17
 8126        vaesenc	xmm8, xmm8, [r15+96]
 8127        vpxor	xmm10, xmm10, xmm11
 8128        vpslldq	xmm2, xmm10, 8
 8129        vpsrldq	xmm10, xmm10, 8
 8130        vaesenc	xmm8, xmm8, [r15+112]
 8131        vpxor	xmm2, xmm2, xmm12
 8132        vpxor	xmm3, xmm1, xmm10
 8133        vmovdqa	xmm0, OWORD PTR L_avx1_aes_gcm_mod2_128
 8134        vpclmulqdq	xmm11, xmm2, xmm0, 16
 8135        vaesenc	xmm8, xmm8, [r15+128]
 8136        vpshufd	xmm10, xmm2, 78
 8137        vpxor	xmm10, xmm10, xmm11
 8138        vpclmulqdq	xmm11, xmm10, xmm0, 16
 8139        vaesenc	xmm8, xmm8, [r15+144]
 8140        vpshufd	xmm10, xmm10, 78
 8141        vpxor	xmm10, xmm10, xmm11
 8142        vpxor	xmm6, xmm10, xmm3
 8143        cmp	r10d, 11
 8144        vmovdqa	xmm9, OWORD PTR [r15+160]
 8145        jl	L_AES_GCM_encrypt_avx1_aesenc_gfmul_last
 8146        vaesenc	xmm8, xmm8, xmm9
 8147        vaesenc	xmm8, xmm8, [r15+176]
 8148        cmp	r10d, 13
 8149        vmovdqa	xmm9, OWORD PTR [r15+192]
 8150        jl	L_AES_GCM_encrypt_avx1_aesenc_gfmul_last
 8151        vaesenc	xmm8, xmm8, xmm9
 8152        vaesenc	xmm8, xmm8, [r15+208]
 8153        vmovdqa	xmm9, OWORD PTR [r15+224]
 8154L_AES_GCM_encrypt_avx1_aesenc_gfmul_last:
 8155        vaesenclast	xmm8, xmm8, xmm9
 8156        vmovdqa	xmm0, xmm13
 8157        vpxor	xmm8, xmm8, xmm0
 8158        vmovdqu	OWORD PTR [rsi+rbx], xmm8
 8159        vpshufb	xmm8, xmm8, OWORD PTR L_avx1_aes_gcm_bswap_mask
 8160        add	ebx, 16
 8161        vpxor	xmm6, xmm6, xmm8
 8162        cmp	ebx, r13d
 8163        jl	L_AES_GCM_encrypt_avx1_last_block_start
 8164L_AES_GCM_encrypt_avx1_last_block_ghash:
 8165        ; ghash_gfmul_red_avx
 8166        vpshufd	xmm9, xmm5, 78
 8167        vpshufd	xmm10, xmm6, 78
 8168        vpclmulqdq	xmm11, xmm6, xmm5, 17
 8169        vpclmulqdq	xmm8, xmm6, xmm5, 0
 8170        vpxor	xmm9, xmm9, xmm5
 8171        vpxor	xmm10, xmm10, xmm6
 8172        vpclmulqdq	xmm9, xmm9, xmm10, 0
 8173        vpxor	xmm9, xmm9, xmm8
 8174        vpxor	xmm9, xmm9, xmm11
 8175        vpslldq	xmm10, xmm9, 8
 8176        vpsrldq	xmm9, xmm9, 8
 8177        vpxor	xmm8, xmm8, xmm10
 8178        vpxor	xmm6, xmm11, xmm9
 8179        vpslld	xmm12, xmm8, 31
 8180        vpslld	xmm13, xmm8, 30
 8181        vpslld	xmm14, xmm8, 25
 8182        vpxor	xmm12, xmm12, xmm13
 8183        vpxor	xmm12, xmm12, xmm14
 8184        vpsrldq	xmm13, xmm12, 4
 8185        vpslldq	xmm12, xmm12, 12
 8186        vpxor	xmm8, xmm8, xmm12
 8187        vpsrld	xmm14, xmm8, 1
 8188        vpsrld	xmm10, xmm8, 2
 8189        vpsrld	xmm9, xmm8, 7
 8190        vpxor	xmm14, xmm14, xmm10
 8191        vpxor	xmm14, xmm14, xmm9
 8192        vpxor	xmm14, xmm14, xmm13
 8193        vpxor	xmm14, xmm14, xmm8
 8194        vpxor	xmm6, xmm6, xmm14
 8195L_AES_GCM_encrypt_avx1_last_block_done:
 8196        mov	ecx, r9d
 8197        mov	edx, ecx
 8198        and	ecx, 15
 8199        jz	L_AES_GCM_encrypt_avx1_aesenc_last15_enc_avx_done
 8200        vmovdqu	xmm4, OWORD PTR [rsp+128]
 8201        vpshufb	xmm4, xmm4, OWORD PTR L_avx1_aes_gcm_bswap_epi64
 8202        vpxor	xmm4, xmm4, [r15]
 8203        vaesenc	xmm4, xmm4, [r15+16]
 8204        vaesenc	xmm4, xmm4, [r15+32]
 8205        vaesenc	xmm4, xmm4, [r15+48]
 8206        vaesenc	xmm4, xmm4, [r15+64]
 8207        vaesenc	xmm4, xmm4, [r15+80]
 8208        vaesenc	xmm4, xmm4, [r15+96]
 8209        vaesenc	xmm4, xmm4, [r15+112]
 8210        vaesenc	xmm4, xmm4, [r15+128]
 8211        vaesenc	xmm4, xmm4, [r15+144]
 8212        cmp	r10d, 11
 8213        vmovdqa	xmm9, OWORD PTR [r15+160]
 8214        jl	L_AES_GCM_encrypt_avx1_aesenc_last15_enc_avx_aesenc_avx_last
 8215        vaesenc	xmm4, xmm4, xmm9
 8216        vaesenc	xmm4, xmm4, [r15+176]
 8217        cmp	r10d, 13
 8218        vmovdqa	xmm9, OWORD PTR [r15+192]
 8219        jl	L_AES_GCM_encrypt_avx1_aesenc_last15_enc_avx_aesenc_avx_last
 8220        vaesenc	xmm4, xmm4, xmm9
 8221        vaesenc	xmm4, xmm4, [r15+208]
 8222        vmovdqa	xmm9, OWORD PTR [r15+224]
 8223L_AES_GCM_encrypt_avx1_aesenc_last15_enc_avx_aesenc_avx_last:
 8224        vaesenclast	xmm4, xmm4, xmm9
 8225        sub	rsp, 16
 8226        xor	ecx, ecx
 8227        vmovdqu	OWORD PTR [rsp], xmm4
 8228L_AES_GCM_encrypt_avx1_aesenc_last15_enc_avx_loop:
 8229        movzx	r13d, BYTE PTR [rdi+rbx]
 8230        xor	r13b, BYTE PTR [rsp+rcx]
 8231        mov	BYTE PTR [rsi+rbx], r13b
 8232        mov	BYTE PTR [rsp+rcx], r13b
 8233        inc	ebx
 8234        inc	ecx
 8235        cmp	ebx, edx
 8236        jl	L_AES_GCM_encrypt_avx1_aesenc_last15_enc_avx_loop
 8237        xor	r13, r13
 8238        cmp	ecx, 16
 8239        je	L_AES_GCM_encrypt_avx1_aesenc_last15_enc_avx_finish_enc
 8240L_AES_GCM_encrypt_avx1_aesenc_last15_enc_avx_byte_loop:
 8241        mov	BYTE PTR [rsp+rcx], r13b
 8242        inc	ecx
 8243        cmp	ecx, 16
 8244        jl	L_AES_GCM_encrypt_avx1_aesenc_last15_enc_avx_byte_loop
 8245L_AES_GCM_encrypt_avx1_aesenc_last15_enc_avx_finish_enc:
 8246        vmovdqu	xmm4, OWORD PTR [rsp]
 8247        add	rsp, 16
 8248        vpshufb	xmm4, xmm4, OWORD PTR L_avx1_aes_gcm_bswap_mask
 8249        vpxor	xmm6, xmm6, xmm4
 8250        ; ghash_gfmul_red_avx
 8251        vpshufd	xmm9, xmm5, 78
 8252        vpshufd	xmm10, xmm6, 78
 8253        vpclmulqdq	xmm11, xmm6, xmm5, 17
 8254        vpclmulqdq	xmm8, xmm6, xmm5, 0
 8255        vpxor	xmm9, xmm9, xmm5
 8256        vpxor	xmm10, xmm10, xmm6
 8257        vpclmulqdq	xmm9, xmm9, xmm10, 0
 8258        vpxor	xmm9, xmm9, xmm8
 8259        vpxor	xmm9, xmm9, xmm11
 8260        vpslldq	xmm10, xmm9, 8
 8261        vpsrldq	xmm9, xmm9, 8
 8262        vpxor	xmm8, xmm8, xmm10
 8263        vpxor	xmm6, xmm11, xmm9
 8264        vpslld	xmm12, xmm8, 31
 8265        vpslld	xmm13, xmm8, 30
 8266        vpslld	xmm14, xmm8, 25
 8267        vpxor	xmm12, xmm12, xmm13
 8268        vpxor	xmm12, xmm12, xmm14
 8269        vpsrldq	xmm13, xmm12, 4
 8270        vpslldq	xmm12, xmm12, 12
 8271        vpxor	xmm8, xmm8, xmm12
 8272        vpsrld	xmm14, xmm8, 1
 8273        vpsrld	xmm10, xmm8, 2
 8274        vpsrld	xmm9, xmm8, 7
 8275        vpxor	xmm14, xmm14, xmm10
 8276        vpxor	xmm14, xmm14, xmm9
 8277        vpxor	xmm14, xmm14, xmm13
 8278        vpxor	xmm14, xmm14, xmm8
 8279        vpxor	xmm6, xmm6, xmm14
 8280L_AES_GCM_encrypt_avx1_aesenc_last15_enc_avx_done:
 8281L_AES_GCM_encrypt_avx1_done_enc:
 8282        mov	edx, r9d
 8283        mov	ecx, r11d
 8284        shl	rdx, 3
 8285        shl	rcx, 3
 8286        vmovq	xmm0, rdx
 8287        vmovq	xmm1, rcx
 8288        vpunpcklqdq	xmm0, xmm0, xmm1
 8289        vpxor	xmm6, xmm6, xmm0
 8290        ; ghash_gfmul_red_avx
 8291        vpshufd	xmm9, xmm5, 78
 8292        vpshufd	xmm10, xmm6, 78
 8293        vpclmulqdq	xmm11, xmm6, xmm5, 17
 8294        vpclmulqdq	xmm8, xmm6, xmm5, 0
 8295        vpxor	xmm9, xmm9, xmm5
 8296        vpxor	xmm10, xmm10, xmm6
 8297        vpclmulqdq	xmm9, xmm9, xmm10, 0
 8298        vpxor	xmm9, xmm9, xmm8
 8299        vpxor	xmm9, xmm9, xmm11
 8300        vpslldq	xmm10, xmm9, 8
 8301        vpsrldq	xmm9, xmm9, 8
 8302        vpxor	xmm8, xmm8, xmm10
 8303        vpxor	xmm6, xmm11, xmm9
 8304        vpslld	xmm12, xmm8, 31
 8305        vpslld	xmm13, xmm8, 30
 8306        vpslld	xmm14, xmm8, 25
 8307        vpxor	xmm12, xmm12, xmm13
 8308        vpxor	xmm12, xmm12, xmm14
 8309        vpsrldq	xmm13, xmm12, 4
 8310        vpslldq	xmm12, xmm12, 12
 8311        vpxor	xmm8, xmm8, xmm12
 8312        vpsrld	xmm14, xmm8, 1
 8313        vpsrld	xmm10, xmm8, 2
 8314        vpsrld	xmm9, xmm8, 7
 8315        vpxor	xmm14, xmm14, xmm10
 8316        vpxor	xmm14, xmm14, xmm9
 8317        vpxor	xmm14, xmm14, xmm13
 8318        vpxor	xmm14, xmm14, xmm8
 8319        vpxor	xmm6, xmm6, xmm14
 8320        vpshufb	xmm6, xmm6, OWORD PTR L_avx1_aes_gcm_bswap_mask
 8321        vmovdqu	xmm0, OWORD PTR [rsp+144]
 8322        vpxor	xmm0, xmm0, xmm6
 8323        cmp	r14d, 16
 8324        je	L_AES_GCM_encrypt_avx1_store_tag_16
 8325        xor	rcx, rcx
 8326        vmovdqu	OWORD PTR [rsp], xmm0
 8327L_AES_GCM_encrypt_avx1_store_tag_loop:
 8328        movzx	r13d, BYTE PTR [rsp+rcx]
 8329        mov	BYTE PTR [r8+rcx], r13b
 8330        inc	ecx
 8331        cmp	ecx, r14d
 8332        jne	L_AES_GCM_encrypt_avx1_store_tag_loop
 8333        jmp	L_AES_GCM_encrypt_avx1_store_tag_done
 8334L_AES_GCM_encrypt_avx1_store_tag_16:
 8335        vmovdqu	OWORD PTR [r8], xmm0
 8336L_AES_GCM_encrypt_avx1_store_tag_done:
 8337        vzeroupper
 8338        vmovdqu	xmm6, OWORD PTR [rsp+160]
 8339        vmovdqu	xmm7, OWORD PTR [rsp+176]
 8340        vmovdqu	xmm8, OWORD PTR [rsp+192]
 8341        vmovdqu	xmm9, OWORD PTR [rsp+208]
 8342        vmovdqu	xmm10, OWORD PTR [rsp+224]
 8343        vmovdqu	xmm11, OWORD PTR [rsp+240]
 8344        vmovdqu	xmm12, OWORD PTR [rsp+256]
 8345        vmovdqu	xmm13, OWORD PTR [rsp+272]
 8346        vmovdqu	xmm14, OWORD PTR [rsp+288]
 8347        vmovdqu	xmm15, OWORD PTR [rsp+304]
 8348        add	rsp, 320
 8349        pop	r15
 8350        pop	r14
 8351        pop	rbx
 8352        pop	r12
 8353        pop	rsi
 8354        pop	rdi
 8355        pop	r13
 8356        ret
 8357AES_GCM_encrypt_avx1 ENDP
 8358_TEXT ENDS
 8359_TEXT SEGMENT READONLY PARA
 8360AES_GCM_decrypt_avx1 PROC
 8361        push	r13
 8362        push	rdi
 8363        push	rsi
 8364        push	r12
 8365        push	rbx
 8366        push	r14
 8367        push	r15
 8368        push	rbp
 8369        mov	rdi, rcx
 8370        mov	rsi, rdx
 8371        mov	r12, r8
 8372        mov	rax, r9
 8373        mov	r8, QWORD PTR [rsp+104]
 8374        mov	r9d, DWORD PTR [rsp+112]
 8375        mov	r11d, DWORD PTR [rsp+120]
 8376        mov	ebx, DWORD PTR [rsp+128]
 8377        mov	r14d, DWORD PTR [rsp+136]
 8378        mov	r15, QWORD PTR [rsp+144]
 8379        mov	r10d, DWORD PTR [rsp+152]
 8380        mov	rbp, QWORD PTR [rsp+160]
 8381        sub	rsp, 328
 8382        vmovdqu	OWORD PTR [rsp+168], xmm6
 8383        vmovdqu	OWORD PTR [rsp+184], xmm7
 8384        vmovdqu	OWORD PTR [rsp+200], xmm8
 8385        vmovdqu	OWORD PTR [rsp+216], xmm9
 8386        vmovdqu	OWORD PTR [rsp+232], xmm10
 8387        vmovdqu	OWORD PTR [rsp+248], xmm11
 8388        vmovdqu	OWORD PTR [rsp+264], xmm12
 8389        vmovdqu	OWORD PTR [rsp+280], xmm13
 8390        vmovdqu	OWORD PTR [rsp+296], xmm14
 8391        vmovdqu	OWORD PTR [rsp+312], xmm15
 8392        vpxor	xmm4, xmm4, xmm4
 8393        vpxor	xmm6, xmm6, xmm6
 8394        cmp	ebx, 12
 8395        mov	edx, ebx
 8396        jne	L_AES_GCM_decrypt_avx1_iv_not_12
 8397        ; # Calculate values when IV is 12 bytes
 8398        ; Set counter based on IV
 8399        mov	ecx, 16777216
 8400        vmovq	xmm4, QWORD PTR [rax]
 8401        vpinsrd	xmm4, xmm4, DWORD PTR [rax+8], 2
 8402        vpinsrd	xmm4, xmm4, ecx, 3
 8403        ; H = Encrypt X(=0) and T = Encrypt counter
 8404        vmovdqa	xmm5, OWORD PTR [r15]
 8405        vpxor	xmm1, xmm4, xmm5
 8406        vmovdqa	xmm7, OWORD PTR [r15+16]
 8407        vaesenc	xmm5, xmm5, xmm7
 8408        vaesenc	xmm1, xmm1, xmm7
 8409        vmovdqa	xmm7, OWORD PTR [r15+32]
 8410        vaesenc	xmm5, xmm5, xmm7
 8411        vaesenc	xmm1, xmm1, xmm7
 8412        vmovdqa	xmm7, OWORD PTR [r15+48]
 8413        vaesenc	xmm5, xmm5, xmm7
 8414        vaesenc	xmm1, xmm1, xmm7
 8415        vmovdqa	xmm7, OWORD PTR [r15+64]
 8416        vaesenc	xmm5, xmm5, xmm7
 8417        vaesenc	xmm1, xmm1, xmm7
 8418        vmovdqa	xmm7, OWORD PTR [r15+80]
 8419        vaesenc	xmm5, xmm5, xmm7
 8420        vaesenc	xmm1, xmm1, xmm7
 8421        vmovdqa	xmm7, OWORD PTR [r15+96]
 8422        vaesenc	xmm5, xmm5, xmm7
 8423        vaesenc	xmm1, xmm1, xmm7
 8424        vmovdqa	xmm7, OWORD PTR [r15+112]
 8425        vaesenc	xmm5, xmm5, xmm7
 8426        vaesenc	xmm1, xmm1, xmm7
 8427        vmovdqa	xmm7, OWORD PTR [r15+128]
 8428        vaesenc	xmm5, xmm5, xmm7
 8429        vaesenc	xmm1, xmm1, xmm7
 8430        vmovdqa	xmm7, OWORD PTR [r15+144]
 8431        vaesenc	xmm5, xmm5, xmm7
 8432        vaesenc	xmm1, xmm1, xmm7
 8433        cmp	r10d, 11
 8434        vmovdqa	xmm7, OWORD PTR [r15+160]
 8435        jl	L_AES_GCM_decrypt_avx1_calc_iv_12_last
 8436        vaesenc	xmm5, xmm5, xmm7
 8437        vaesenc	xmm1, xmm1, xmm7
 8438        vmovdqa	xmm7, OWORD PTR [r15+176]
 8439        vaesenc	xmm5, xmm5, xmm7
 8440        vaesenc	xmm1, xmm1, xmm7
 8441        cmp	r10d, 13
 8442        vmovdqa	xmm7, OWORD PTR [r15+192]
 8443        jl	L_AES_GCM_decrypt_avx1_calc_iv_12_last
 8444        vaesenc	xmm5, xmm5, xmm7
 8445        vaesenc	xmm1, xmm1, xmm7
 8446        vmovdqa	xmm7, OWORD PTR [r15+208]
 8447        vaesenc	xmm5, xmm5, xmm7
 8448        vaesenc	xmm1, xmm1, xmm7
 8449        vmovdqa	xmm7, OWORD PTR [r15+224]
 8450L_AES_GCM_decrypt_avx1_calc_iv_12_last:
 8451        vaesenclast	xmm5, xmm5, xmm7
 8452        vaesenclast	xmm1, xmm1, xmm7
 8453        vpshufb	xmm5, xmm5, OWORD PTR L_avx1_aes_gcm_bswap_mask
 8454        vmovdqu	OWORD PTR [rsp+144], xmm1
 8455        jmp	L_AES_GCM_decrypt_avx1_iv_done
 8456L_AES_GCM_decrypt_avx1_iv_not_12:
 8457        ; Calculate values when IV is not 12 bytes
 8458        ; H = Encrypt X(=0)
 8459        vmovdqa	xmm5, OWORD PTR [r15]
 8460        vaesenc	xmm5, xmm5, [r15+16]
 8461        vaesenc	xmm5, xmm5, [r15+32]
 8462        vaesenc	xmm5, xmm5, [r15+48]
 8463        vaesenc	xmm5, xmm5, [r15+64]
 8464        vaesenc	xmm5, xmm5, [r15+80]
 8465        vaesenc	xmm5, xmm5, [r15+96]
 8466        vaesenc	xmm5, xmm5, [r15+112]
 8467        vaesenc	xmm5, xmm5, [r15+128]
 8468        vaesenc	xmm5, xmm5, [r15+144]
 8469        cmp	r10d, 11
 8470        vmovdqa	xmm9, OWORD PTR [r15+160]
 8471        jl	L_AES_GCM_decrypt_avx1_calc_iv_1_aesenc_avx_last
 8472        vaesenc	xmm5, xmm5, xmm9
 8473        vaesenc	xmm5, xmm5, [r15+176]
 8474        cmp	r10d, 13
 8475        vmovdqa	xmm9, OWORD PTR [r15+192]
 8476        jl	L_AES_GCM_decrypt_avx1_calc_iv_1_aesenc_avx_last
 8477        vaesenc	xmm5, xmm5, xmm9
 8478        vaesenc	xmm5, xmm5, [r15+208]
 8479        vmovdqa	xmm9, OWORD PTR [r15+224]
 8480L_AES_GCM_decrypt_avx1_calc_iv_1_aesenc_avx_last:
 8481        vaesenclast	xmm5, xmm5, xmm9
 8482        vpshufb	xmm5, xmm5, OWORD PTR L_avx1_aes_gcm_bswap_mask
 8483        ; Calc counter
 8484        ; Initialization vector
 8485        cmp	edx, 0
 8486        mov	rcx, 0
 8487        je	L_AES_GCM_decrypt_avx1_calc_iv_done
 8488        cmp	edx, 16
 8489        jl	L_AES_GCM_decrypt_avx1_calc_iv_lt16
 8490        and	edx, 4294967280
 8491L_AES_GCM_decrypt_avx1_calc_iv_16_loop:
 8492        vmovdqu	xmm8, OWORD PTR [rax+rcx]
 8493        vpshufb	xmm8, xmm8, OWORD PTR L_avx1_aes_gcm_bswap_mask
 8494        vpxor	xmm4, xmm4, xmm8
 8495        ; ghash_gfmul_avx
 8496        vpshufd	xmm1, xmm4, 78
 8497        vpshufd	xmm2, xmm5, 78
 8498        vpclmulqdq	xmm3, xmm5, xmm4, 17
 8499        vpclmulqdq	xmm0, xmm5, xmm4, 0
 8500        vpxor	xmm1, xmm1, xmm4
 8501        vpxor	xmm2, xmm2, xmm5
 8502        vpclmulqdq	xmm1, xmm1, xmm2, 0
 8503        vpxor	xmm1, xmm1, xmm0
 8504        vpxor	xmm1, xmm1, xmm3
 8505        vmovdqa	xmm7, xmm0
 8506        vmovdqa	xmm4, xmm3
 8507        vpslldq	xmm2, xmm1, 8
 8508        vpsrldq	xmm1, xmm1, 8
 8509        vpxor	xmm7, xmm7, xmm2
 8510        vpxor	xmm4, xmm4, xmm1
 8511        vpsrld	xmm0, xmm7, 31
 8512        vpsrld	xmm1, xmm4, 31
 8513        vpslld	xmm7, xmm7, 1
 8514        vpslld	xmm4, xmm4, 1
 8515        vpsrldq	xmm2, xmm0, 12
 8516        vpslldq	xmm0, xmm0, 4
 8517        vpslldq	xmm1, xmm1, 4
 8518        vpor	xmm4, xmm4, xmm2
 8519        vpor	xmm7, xmm7, xmm0
 8520        vpor	xmm4, xmm4, xmm1
 8521        vpslld	xmm0, xmm7, 31
 8522        vpslld	xmm1, xmm7, 30
 8523        vpslld	xmm2, xmm7, 25
 8524        vpxor	xmm0, xmm0, xmm1
 8525        vpxor	xmm0, xmm0, xmm2
 8526        vmovdqa	xmm1, xmm0
 8527        vpsrldq	xmm1, xmm1, 4
 8528        vpslldq	xmm0, xmm0, 12
 8529        vpxor	xmm7, xmm7, xmm0
 8530        vpsrld	xmm2, xmm7, 1
 8531        vpsrld	xmm3, xmm7, 2
 8532        vpsrld	xmm0, xmm7, 7
 8533        vpxor	xmm2, xmm2, xmm3
 8534        vpxor	xmm2, xmm2, xmm0
 8535        vpxor	xmm2, xmm2, xmm1
 8536        vpxor	xmm2, xmm2, xmm7
 8537        vpxor	xmm4, xmm4, xmm2
 8538        add	ecx, 16
 8539        cmp	ecx, edx
 8540        jl	L_AES_GCM_decrypt_avx1_calc_iv_16_loop
 8541        mov	edx, ebx
 8542        cmp	ecx, edx
 8543        je	L_AES_GCM_decrypt_avx1_calc_iv_done
 8544L_AES_GCM_decrypt_avx1_calc_iv_lt16:
 8545        sub	rsp, 16
 8546        vpxor	xmm8, xmm8, xmm8
 8547        xor	ebx, ebx
 8548        vmovdqu	OWORD PTR [rsp], xmm8
 8549L_AES_GCM_decrypt_avx1_calc_iv_loop:
 8550        movzx	r13d, BYTE PTR [rax+rcx]
 8551        mov	BYTE PTR [rsp+rbx], r13b
 8552        inc	ecx
 8553        inc	ebx
 8554        cmp	ecx, edx
 8555        jl	L_AES_GCM_decrypt_avx1_calc_iv_loop
 8556        vmovdqu	xmm8, OWORD PTR [rsp]
 8557        add	rsp, 16
 8558        vpshufb	xmm8, xmm8, OWORD PTR L_avx1_aes_gcm_bswap_mask
 8559        vpxor	xmm4, xmm4, xmm8
 8560        ; ghash_gfmul_avx
 8561        vpshufd	xmm1, xmm4, 78
 8562        vpshufd	xmm2, xmm5, 78
 8563        vpclmulqdq	xmm3, xmm5, xmm4, 17
 8564        vpclmulqdq	xmm0, xmm5, xmm4, 0
 8565        vpxor	xmm1, xmm1, xmm4
 8566        vpxor	xmm2, xmm2, xmm5
 8567        vpclmulqdq	xmm1, xmm1, xmm2, 0
 8568        vpxor	xmm1, xmm1, xmm0
 8569        vpxor	xmm1, xmm1, xmm3
 8570        vmovdqa	xmm7, xmm0
 8571        vmovdqa	xmm4, xmm3
 8572        vpslldq	xmm2, xmm1, 8
 8573        vpsrldq	xmm1, xmm1, 8
 8574        vpxor	xmm7, xmm7, xmm2
 8575        vpxor	xmm4, xmm4, xmm1
 8576        vpsrld	xmm0, xmm7, 31
 8577        vpsrld	xmm1, xmm4, 31
 8578        vpslld	xmm7, xmm7, 1
 8579        vpslld	xmm4, xmm4, 1
 8580        vpsrldq	xmm2, xmm0, 12
 8581        vpslldq	xmm0, xmm0, 4
 8582        vpslldq	xmm1, xmm1, 4
 8583        vpor	xmm4, xmm4, xmm2
 8584        vpor	xmm7, xmm7, xmm0
 8585        vpor	xmm4, xmm4, xmm1
 8586        vpslld	xmm0, xmm7, 31
 8587        vpslld	xmm1, xmm7, 30
 8588        vpslld	xmm2, xmm7, 25
 8589        vpxor	xmm0, xmm0, xmm1
 8590        vpxor	xmm0, xmm0, xmm2
 8591        vmovdqa	xmm1, xmm0
 8592        vpsrldq	xmm1, xmm1, 4
 8593        vpslldq	xmm0, xmm0, 12
 8594        vpxor	xmm7, xmm7, xmm0
 8595        vpsrld	xmm2, xmm7, 1
 8596        vpsrld	xmm3, xmm7, 2
 8597        vpsrld	xmm0, xmm7, 7
 8598        vpxor	xmm2, xmm2, xmm3
 8599        vpxor	xmm2, xmm2, xmm0
 8600        vpxor	xmm2, xmm2, xmm1
 8601        vpxor	xmm2, xmm2, xmm7
 8602        vpxor	xmm4, xmm4, xmm2
 8603L_AES_GCM_decrypt_avx1_calc_iv_done:
 8604        ; T = Encrypt counter
 8605        vpxor	xmm0, xmm0, xmm0
 8606        shl	edx, 3
 8607        vmovq	xmm0, rdx
 8608        vpxor	xmm4, xmm4, xmm0
 8609        ; ghash_gfmul_avx
 8610        vpshufd	xmm1, xmm4, 78
 8611        vpshufd	xmm2, xmm5, 78
 8612        vpclmulqdq	xmm3, xmm5, xmm4, 17
 8613        vpclmulqdq	xmm0, xmm5, xmm4, 0
 8614        vpxor	xmm1, xmm1, xmm4
 8615        vpxor	xmm2, xmm2, xmm5
 8616        vpclmulqdq	xmm1, xmm1, xmm2, 0
 8617        vpxor	xmm1, xmm1, xmm0
 8618        vpxor	xmm1, xmm1, xmm3
 8619        vmovdqa	xmm7, xmm0
 8620        vmovdqa	xmm4, xmm3
 8621        vpslldq	xmm2, xmm1, 8
 8622        vpsrldq	xmm1, xmm1, 8
 8623        vpxor	xmm7, xmm7, xmm2
 8624        vpxor	xmm4, xmm4, xmm1
 8625        vpsrld	xmm0, xmm7, 31
 8626        vpsrld	xmm1, xmm4, 31
 8627        vpslld	xmm7, xmm7, 1
 8628        vpslld	xmm4, xmm4, 1
 8629        vpsrldq	xmm2, xmm0, 12
 8630        vpslldq	xmm0, xmm0, 4
 8631        vpslldq	xmm1, xmm1, 4
 8632        vpor	xmm4, xmm4, xmm2
 8633        vpor	xmm7, xmm7, xmm0
 8634        vpor	xmm4, xmm4, xmm1
 8635        vpslld	xmm0, xmm7, 31
 8636        vpslld	xmm1, xmm7, 30
 8637        vpslld	xmm2, xmm7, 25
 8638        vpxor	xmm0, xmm0, xmm1
 8639        vpxor	xmm0, xmm0, xmm2
 8640        vmovdqa	xmm1, xmm0
 8641        vpsrldq	xmm1, xmm1, 4
 8642        vpslldq	xmm0, xmm0, 12
 8643        vpxor	xmm7, xmm7, xmm0
 8644        vpsrld	xmm2, xmm7, 1
 8645        vpsrld	xmm3, xmm7, 2
 8646        vpsrld	xmm0, xmm7, 7
 8647        vpxor	xmm2, xmm2, xmm3
 8648        vpxor	xmm2, xmm2, xmm0
 8649        vpxor	xmm2, xmm2, xmm1
 8650        vpxor	xmm2, xmm2, xmm7
 8651        vpxor	xmm4, xmm4, xmm2
 8652        vpshufb	xmm4, xmm4, OWORD PTR L_avx1_aes_gcm_bswap_mask
 8653        ;   Encrypt counter
 8654        vmovdqa	xmm8, OWORD PTR [r15]
 8655        vpxor	xmm8, xmm8, xmm4
 8656        vaesenc	xmm8, xmm8, [r15+16]
 8657        vaesenc	xmm8, xmm8, [r15+32]
 8658        vaesenc	xmm8, xmm8, [r15+48]
 8659        vaesenc	xmm8, xmm8, [r15+64]
 8660        vaesenc	xmm8, xmm8, [r15+80]
 8661        vaesenc	xmm8, xmm8, [r15+96]
 8662        vaesenc	xmm8, xmm8, [r15+112]
 8663        vaesenc	xmm8, xmm8, [r15+128]
 8664        vaesenc	xmm8, xmm8, [r15+144]
 8665        cmp	r10d, 11
 8666        vmovdqa	xmm9, OWORD PTR [r15+160]
 8667        jl	L_AES_GCM_decrypt_avx1_calc_iv_2_aesenc_avx_last
 8668        vaesenc	xmm8, xmm8, xmm9
 8669        vaesenc	xmm8, xmm8, [r15+176]
 8670        cmp	r10d, 13
 8671        vmovdqa	xmm9, OWORD PTR [r15+192]
 8672        jl	L_AES_GCM_decrypt_avx1_calc_iv_2_aesenc_avx_last
 8673        vaesenc	xmm8, xmm8, xmm9
 8674        vaesenc	xmm8, xmm8, [r15+208]
 8675        vmovdqa	xmm9, OWORD PTR [r15+224]
 8676L_AES_GCM_decrypt_avx1_calc_iv_2_aesenc_avx_last:
 8677        vaesenclast	xmm8, xmm8, xmm9
 8678        vmovdqu	OWORD PTR [rsp+144], xmm8
 8679L_AES_GCM_decrypt_avx1_iv_done:
 8680        ; Additional authentication data
 8681        mov	edx, r11d
 8682        cmp	edx, 0
 8683        je	L_AES_GCM_decrypt_avx1_calc_aad_done
 8684        xor	ecx, ecx
 8685        cmp	edx, 16
 8686        jl	L_AES_GCM_decrypt_avx1_calc_aad_lt16
 8687        and	edx, 4294967280
 8688L_AES_GCM_decrypt_avx1_calc_aad_16_loop:
 8689        vmovdqu	xmm8, OWORD PTR [r12+rcx]
 8690        vpshufb	xmm8, xmm8, OWORD PTR L_avx1_aes_gcm_bswap_mask
 8691        vpxor	xmm6, xmm6, xmm8
 8692        ; ghash_gfmul_avx
 8693        vpshufd	xmm1, xmm6, 78
 8694        vpshufd	xmm2, xmm5, 78
 8695        vpclmulqdq	xmm3, xmm5, xmm6, 17
 8696        vpclmulqdq	xmm0, xmm5, xmm6, 0
 8697        vpxor	xmm1, xmm1, xmm6
 8698        vpxor	xmm2, xmm2, xmm5
 8699        vpclmulqdq	xmm1, xmm1, xmm2, 0
 8700        vpxor	xmm1, xmm1, xmm0
 8701        vpxor	xmm1, xmm1, xmm3
 8702        vmovdqa	xmm7, xmm0
 8703        vmovdqa	xmm6, xmm3
 8704        vpslldq	xmm2, xmm1, 8
 8705        vpsrldq	xmm1, xmm1, 8
 8706        vpxor	xmm7, xmm7, xmm2
 8707        vpxor	xmm6, xmm6, xmm1
 8708        vpsrld	xmm0, xmm7, 31
 8709        vpsrld	xmm1, xmm6, 31
 8710        vpslld	xmm7, xmm7, 1
 8711        vpslld	xmm6, xmm6, 1
 8712        vpsrldq	xmm2, xmm0, 12
 8713        vpslldq	xmm0, xmm0, 4
 8714        vpslldq	xmm1, xmm1, 4
 8715        vpor	xmm6, xmm6, xmm2
 8716        vpor	xmm7, xmm7, xmm0
 8717        vpor	xmm6, xmm6, xmm1
 8718        vpslld	xmm0, xmm7, 31
 8719        vpslld	xmm1, xmm7, 30
 8720        vpslld	xmm2, xmm7, 25
 8721        vpxor	xmm0, xmm0, xmm1
 8722        vpxor	xmm0, xmm0, xmm2
 8723        vmovdqa	xmm1, xmm0
 8724        vpsrldq	xmm1, xmm1, 4
 8725        vpslldq	xmm0, xmm0, 12
 8726        vpxor	xmm7, xmm7, xmm0
 8727        vpsrld	xmm2, xmm7, 1
 8728        vpsrld	xmm3, xmm7, 2
 8729        vpsrld	xmm0, xmm7, 7
 8730        vpxor	xmm2, xmm2, xmm3
 8731        vpxor	xmm2, xmm2, xmm0
 8732        vpxor	xmm2, xmm2, xmm1
 8733        vpxor	xmm2, xmm2, xmm7
 8734        vpxor	xmm6, xmm6, xmm2
 8735        add	ecx, 16
 8736        cmp	ecx, edx
 8737        jl	L_AES_GCM_decrypt_avx1_calc_aad_16_loop
 8738        mov	edx, r11d
 8739        cmp	ecx, edx
 8740        je	L_AES_GCM_decrypt_avx1_calc_aad_done
 8741L_AES_GCM_decrypt_avx1_calc_aad_lt16:
 8742        sub	rsp, 16
 8743        vpxor	xmm8, xmm8, xmm8
 8744        xor	ebx, ebx
 8745        vmovdqu	OWORD PTR [rsp], xmm8
 8746L_AES_GCM_decrypt_avx1_calc_aad_loop:
 8747        movzx	r13d, BYTE PTR [r12+rcx]
 8748        mov	BYTE PTR [rsp+rbx], r13b
 8749        inc	ecx
 8750        inc	ebx
 8751        cmp	ecx, edx
 8752        jl	L_AES_GCM_decrypt_avx1_calc_aad_loop
 8753        vmovdqu	xmm8, OWORD PTR [rsp]
 8754        add	rsp, 16
 8755        vpshufb	xmm8, xmm8, OWORD PTR L_avx1_aes_gcm_bswap_mask
 8756        vpxor	xmm6, xmm6, xmm8
 8757        ; ghash_gfmul_avx
 8758        vpshufd	xmm1, xmm6, 78
 8759        vpshufd	xmm2, xmm5, 78
 8760        vpclmulqdq	xmm3, xmm5, xmm6, 17
 8761        vpclmulqdq	xmm0, xmm5, xmm6, 0
 8762        vpxor	xmm1, xmm1, xmm6
 8763        vpxor	xmm2, xmm2, xmm5
 8764        vpclmulqdq	xmm1, xmm1, xmm2, 0
 8765        vpxor	xmm1, xmm1, xmm0
 8766        vpxor	xmm1, xmm1, xmm3
 8767        vmovdqa	xmm7, xmm0
 8768        vmovdqa	xmm6, xmm3
 8769        vpslldq	xmm2, xmm1, 8
 8770        vpsrldq	xmm1, xmm1, 8
 8771        vpxor	xmm7, xmm7, xmm2
 8772        vpxor	xmm6, xmm6, xmm1
 8773        vpsrld	xmm0, xmm7, 31
 8774        vpsrld	xmm1, xmm6, 31
 8775        vpslld	xmm7, xmm7, 1
 8776        vpslld	xmm6, xmm6, 1
 8777        vpsrldq	xmm2, xmm0, 12
 8778        vpslldq	xmm0, xmm0, 4
 8779        vpslldq	xmm1, xmm1, 4
 8780        vpor	xmm6, xmm6, xmm2
 8781        vpor	xmm7, xmm7, xmm0
 8782        vpor	xmm6, xmm6, xmm1
 8783        vpslld	xmm0, xmm7, 31
 8784        vpslld	xmm1, xmm7, 30
 8785        vpslld	xmm2, xmm7, 25
 8786        vpxor	xmm0, xmm0, xmm1
 8787        vpxor	xmm0, xmm0, xmm2
 8788        vmovdqa	xmm1, xmm0
 8789        vpsrldq	xmm1, xmm1, 4
 8790        vpslldq	xmm0, xmm0, 12
 8791        vpxor	xmm7, xmm7, xmm0
 8792        vpsrld	xmm2, xmm7, 1
 8793        vpsrld	xmm3, xmm7, 2
 8794        vpsrld	xmm0, xmm7, 7
 8795        vpxor	xmm2, xmm2, xmm3
 8796        vpxor	xmm2, xmm2, xmm0
 8797        vpxor	xmm2, xmm2, xmm1
 8798        vpxor	xmm2, xmm2, xmm7
 8799        vpxor	xmm6, xmm6, xmm2
 8800L_AES_GCM_decrypt_avx1_calc_aad_done:
 8801        ; Calculate counter and H
 8802        vpsrlq	xmm9, xmm5, 63
 8803        vpsllq	xmm8, xmm5, 1
 8804        vpslldq	xmm9, xmm9, 8
 8805        vpor	xmm8, xmm8, xmm9
 8806        vpshufd	xmm5, xmm5, 255
 8807        vpsrad	xmm5, xmm5, 31
 8808        vpshufb	xmm4, xmm4, OWORD PTR L_avx1_aes_gcm_bswap_epi64
 8809        vpand	xmm5, xmm5, OWORD PTR L_avx1_aes_gcm_mod2_128
 8810        vpaddd	xmm4, xmm4, OWORD PTR L_avx1_aes_gcm_one
 8811        vpxor	xmm5, xmm5, xmm8
 8812        vmovdqu	OWORD PTR [rsp+128], xmm4
 8813        xor	ebx, ebx
 8814        cmp	r9d, 128
 8815        mov	r13d, r9d
 8816        jl	L_AES_GCM_decrypt_avx1_done_128
 8817        and	r13d, 4294967168
 8818        vmovdqa	xmm2, xmm6
 8819        ; H ^ 1
 8820        vmovdqu	OWORD PTR [rsp], xmm5
 8821        ; H ^ 2
 8822        vpclmulqdq	xmm8, xmm5, xmm5, 0
 8823        vpclmulqdq	xmm0, xmm5, xmm5, 17
 8824        vpslld	xmm12, xmm8, 31
 8825        vpslld	xmm13, xmm8, 30
 8826        vpslld	xmm14, xmm8, 25
 8827        vpxor	xmm12, xmm12, xmm13
 8828        vpxor	xmm12, xmm12, xmm14
 8829        vpsrldq	xmm13, xmm12, 4
 8830        vpslldq	xmm12, xmm12, 12
 8831        vpxor	xmm8, xmm8, xmm12
 8832        vpsrld	xmm14, xmm8, 1
 8833        vpsrld	xmm10, xmm8, 2
 8834        vpsrld	xmm9, xmm8, 7
 8835        vpxor	xmm14, xmm14, xmm10
 8836        vpxor	xmm14, xmm14, xmm9
 8837        vpxor	xmm14, xmm14, xmm13
 8838        vpxor	xmm14, xmm14, xmm8
 8839        vpxor	xmm0, xmm0, xmm14
 8840        vmovdqu	OWORD PTR [rsp+16], xmm0
 8841        ; H ^ 3
 8842        ; ghash_gfmul_red_avx
 8843        vpshufd	xmm9, xmm5, 78
 8844        vpshufd	xmm10, xmm0, 78
 8845        vpclmulqdq	xmm11, xmm0, xmm5, 17
 8846        vpclmulqdq	xmm8, xmm0, xmm5, 0
 8847        vpxor	xmm9, xmm9, xmm5
 8848        vpxor	xmm10, xmm10, xmm0
 8849        vpclmulqdq	xmm9, xmm9, xmm10, 0
 8850        vpxor	xmm9, xmm9, xmm8
 8851        vpxor	xmm9, xmm9, xmm11
 8852        vpslldq	xmm10, xmm9, 8
 8853        vpsrldq	xmm9, xmm9, 8
 8854        vpxor	xmm8, xmm8, xmm10
 8855        vpxor	xmm1, xmm11, xmm9
 8856        vpslld	xmm12, xmm8, 31
 8857        vpslld	xmm13, xmm8, 30
 8858        vpslld	xmm14, xmm8, 25
 8859        vpxor	xmm12, xmm12, xmm13
 8860        vpxor	xmm12, xmm12, xmm14
 8861        vpsrldq	xmm13, xmm12, 4
 8862        vpslldq	xmm12, xmm12, 12
 8863        vpxor	xmm8, xmm8, xmm12
 8864        vpsrld	xmm14, xmm8, 1
 8865        vpsrld	xmm10, xmm8, 2
 8866        vpsrld	xmm9, xmm8, 7
 8867        vpxor	xmm14, xmm14, xmm10
 8868        vpxor	xmm14, xmm14, xmm9
 8869        vpxor	xmm14, xmm14, xmm13
 8870        vpxor	xmm14, xmm14, xmm8
 8871        vpxor	xmm1, xmm1, xmm14
 8872        vmovdqu	OWORD PTR [rsp+32], xmm1
 8873        ; H ^ 4
 8874        vpclmulqdq	xmm8, xmm0, xmm0, 0
 8875        vpclmulqdq	xmm3, xmm0, xmm0, 17
 8876        vpslld	xmm12, xmm8, 31
 8877        vpslld	xmm13, xmm8, 30
 8878        vpslld	xmm14, xmm8, 25
 8879        vpxor	xmm12, xmm12, xmm13
 8880        vpxor	xmm12, xmm12, xmm14
 8881        vpsrldq	xmm13, xmm12, 4
 8882        vpslldq	xmm12, xmm12, 12
 8883        vpxor	xmm8, xmm8, xmm12
 8884        vpsrld	xmm14, xmm8, 1
 8885        vpsrld	xmm10, xmm8, 2
 8886        vpsrld	xmm9, xmm8, 7
 8887        vpxor	xmm14, xmm14, xmm10
 8888        vpxor	xmm14, xmm14, xmm9
 8889        vpxor	xmm14, xmm14, xmm13
 8890        vpxor	xmm14, xmm14, xmm8
 8891        vpxor	xmm3, xmm3, xmm14
 8892        vmovdqu	OWORD PTR [rsp+48], xmm3
 8893        ; H ^ 5
 8894        ; ghash_gfmul_red_avx
 8895        vpshufd	xmm9, xmm0, 78
 8896        vpshufd	xmm10, xmm1, 78
 8897        vpclmulqdq	xmm11, xmm1, xmm0, 17
 8898        vpclmulqdq	xmm8, xmm1, xmm0, 0
 8899        vpxor	xmm9, xmm9, xmm0
 8900        vpxor	xmm10, xmm10, xmm1
 8901        vpclmulqdq	xmm9, xmm9, xmm10, 0
 8902        vpxor	xmm9, xmm9, xmm8
 8903        vpxor	xmm9, xmm9, xmm11
 8904        vpslldq	xmm10, xmm9, 8
 8905        vpsrldq	xmm9, xmm9, 8
 8906        vpxor	xmm8, xmm8, xmm10
 8907        vpxor	xmm7, xmm11, xmm9
 8908        vpslld	xmm12, xmm8, 31
 8909        vpslld	xmm13, xmm8, 30
 8910        vpslld	xmm14, xmm8, 25
 8911        vpxor	xmm12, xmm12, xmm13
 8912        vpxor	xmm12, xmm12, xmm14
 8913        vpsrldq	xmm13, xmm12, 4
 8914        vpslldq	xmm12, xmm12, 12
 8915        vpxor	xmm8, xmm8, xmm12
 8916        vpsrld	xmm14, xmm8, 1
 8917        vpsrld	xmm10, xmm8, 2
 8918        vpsrld	xmm9, xmm8, 7
 8919        vpxor	xmm14, xmm14, xmm10
 8920        vpxor	xmm14, xmm14, xmm9
 8921        vpxor	xmm14, xmm14, xmm13
 8922        vpxor	xmm14, xmm14, xmm8
 8923        vpxor	xmm7, xmm7, xmm14
 8924        vmovdqu	OWORD PTR [rsp+64], xmm7
 8925        ; H ^ 6
 8926        vpclmulqdq	xmm8, xmm1, xmm1, 0
 8927        vpclmulqdq	xmm7, xmm1, xmm1, 17
 8928        vpslld	xmm12, xmm8, 31
 8929        vpslld	xmm13, xmm8, 30
 8930        vpslld	xmm14, xmm8, 25
 8931        vpxor	xmm12, xmm12, xmm13
 8932        vpxor	xmm12, xmm12, xmm14
 8933        vpsrldq	xmm13, xmm12, 4
 8934        vpslldq	xmm12, xmm12, 12
 8935        vpxor	xmm8, xmm8, xmm12
 8936        vpsrld	xmm14, xmm8, 1
 8937        vpsrld	xmm10, xmm8, 2
 8938        vpsrld	xmm9, xmm8, 7
 8939        vpxor	xmm14, xmm14, xmm10
 8940        vpxor	xmm14, xmm14, xmm9
 8941        vpxor	xmm14, xmm14, xmm13
 8942        vpxor	xmm14, xmm14, xmm8
 8943        vpxor	xmm7, xmm7, xmm14
 8944        vmovdqu	OWORD PTR [rsp+80], xmm7
 8945        ; H ^ 7
 8946        ; ghash_gfmul_red_avx
 8947        vpshufd	xmm9, xmm1, 78
 8948        vpshufd	xmm10, xmm3, 78
 8949        vpclmulqdq	xmm11, xmm3, xmm1, 17
 8950        vpclmulqdq	xmm8, xmm3, xmm1, 0
 8951        vpxor	xmm9, xmm9, xmm1
 8952        vpxor	xmm10, xmm10, xmm3
 8953        vpclmulqdq	xmm9, xmm9, xmm10, 0
 8954        vpxor	xmm9, xmm9, xmm8
 8955        vpxor	xmm9, xmm9, xmm11
 8956        vpslldq	xmm10, xmm9, 8
 8957        vpsrldq	xmm9, xmm9, 8
 8958        vpxor	xmm8, xmm8, xmm10
 8959        vpxor	xmm7, xmm11, xmm9
 8960        vpslld	xmm12, xmm8, 31
 8961        vpslld	xmm13, xmm8, 30
 8962        vpslld	xmm14, xmm8, 25
 8963        vpxor	xmm12, xmm12, xmm13
 8964        vpxor	xmm12, xmm12, xmm14
 8965        vpsrldq	xmm13, xmm12, 4
 8966        vpslldq	xmm12, xmm12, 12
 8967        vpxor	xmm8, xmm8, xmm12
 8968        vpsrld	xmm14, xmm8, 1
 8969        vpsrld	xmm10, xmm8, 2
 8970        vpsrld	xmm9, xmm8, 7
 8971        vpxor	xmm14, xmm14, xmm10
 8972        vpxor	xmm14, xmm14, xmm9
 8973        vpxor	xmm14, xmm14, xmm13
 8974        vpxor	xmm14, xmm14, xmm8
 8975        vpxor	xmm7, xmm7, xmm14
 8976        vmovdqu	OWORD PTR [rsp+96], xmm7
 8977        ; H ^ 8
 8978        vpclmulqdq	xmm8, xmm3, xmm3, 0
 8979        vpclmulqdq	xmm7, xmm3, xmm3, 17
 8980        vpslld	xmm12, xmm8, 31
 8981        vpslld	xmm13, xmm8, 30
 8982        vpslld	xmm14, xmm8, 25
 8983        vpxor	xmm12, xmm12, xmm13
 8984        vpxor	xmm12, xmm12, xmm14
 8985        vpsrldq	xmm13, xmm12, 4
 8986        vpslldq	xmm12, xmm12, 12
 8987        vpxor	xmm8, xmm8, xmm12
 8988        vpsrld	xmm14, xmm8, 1
 8989        vpsrld	xmm10, xmm8, 2
 8990        vpsrld	xmm9, xmm8, 7
 8991        vpxor	xmm14, xmm14, xmm10
 8992        vpxor	xmm14, xmm14, xmm9
 8993        vpxor	xmm14, xmm14, xmm13
 8994        vpxor	xmm14, xmm14, xmm8
 8995        vpxor	xmm7, xmm7, xmm14
 8996        vmovdqu	OWORD PTR [rsp+112], xmm7
 8997L_AES_GCM_decrypt_avx1_ghash_128:
 8998        lea	rcx, QWORD PTR [rdi+rbx]
 8999        lea	rdx, QWORD PTR [rsi+rbx]
 9000        vmovdqu	xmm0, OWORD PTR [rsp+128]
 9001        vmovdqa	xmm1, OWORD PTR L_avx1_aes_gcm_bswap_epi64
 9002        vpshufb	xmm8, xmm0, xmm1
 9003        vpaddd	xmm9, xmm0, OWORD PTR L_avx1_aes_gcm_one
 9004        vpshufb	xmm9, xmm9, xmm1
 9005        vpaddd	xmm10, xmm0, OWORD PTR L_avx1_aes_gcm_two
 9006        vpshufb	xmm10, xmm10, xmm1
 9007        vpaddd	xmm11, xmm0, OWORD PTR L_avx1_aes_gcm_three
 9008        vpshufb	xmm11, xmm11, xmm1
 9009        vpaddd	xmm12, xmm0, OWORD PTR L_avx1_aes_gcm_four
 9010        vpshufb	xmm12, xmm12, xmm1
 9011        vpaddd	xmm13, xmm0, OWORD PTR L_avx1_aes_gcm_five
 9012        vpshufb	xmm13, xmm13, xmm1
 9013        vpaddd	xmm14, xmm0, OWORD PTR L_avx1_aes_gcm_six
 9014        vpshufb	xmm14, xmm14, xmm1
 9015        vpaddd	xmm15, xmm0, OWORD PTR L_avx1_aes_gcm_seven
 9016        vpshufb	xmm15, xmm15, xmm1
 9017        vpaddd	xmm0, xmm0, OWORD PTR L_avx1_aes_gcm_eight
 9018        vmovdqa	xmm7, OWORD PTR [r15]
 9019        vmovdqu	OWORD PTR [rsp+128], xmm0
 9020        vpxor	xmm8, xmm8, xmm7
 9021        vpxor	xmm9, xmm9, xmm7
 9022        vpxor	xmm10, xmm10, xmm7
 9023        vpxor	xmm11, xmm11, xmm7
 9024        vpxor	xmm12, xmm12, xmm7
 9025        vpxor	xmm13, xmm13, xmm7
 9026        vpxor	xmm14, xmm14, xmm7
 9027        vpxor	xmm15, xmm15, xmm7
 9028        vmovdqu	xmm7, OWORD PTR [rsp+112]
 9029        vmovdqu	xmm0, OWORD PTR [rcx]
 9030        vaesenc	xmm8, xmm8, [r15+16]
 9031        vpshufb	xmm0, xmm0, OWORD PTR L_avx1_aes_gcm_bswap_mask
 9032        vpxor	xmm0, xmm0, xmm2
 9033        vpshufd	xmm1, xmm7, 78
 9034        vpshufd	xmm5, xmm0, 78
 9035        vpxor	xmm1, xmm1, xmm7
 9036        vpxor	xmm5, xmm5, xmm0
 9037        vpclmulqdq	xmm3, xmm0, xmm7, 17
 9038        vaesenc	xmm9, xmm9, [r15+16]
 9039        vaesenc	xmm10, xmm10, [r15+16]
 9040        vpclmulqdq	xmm2, xmm0, xmm7, 0
 9041        vaesenc	xmm11, xmm11, [r15+16]
 9042        vaesenc	xmm12, xmm12, [r15+16]
 9043        vpclmulqdq	xmm1, xmm1, xmm5, 0
 9044        vaesenc	xmm13, xmm13, [r15+16]
 9045        vaesenc	xmm14, xmm14, [r15+16]
 9046        vaesenc	xmm15, xmm15, [r15+16]
 9047        vpxor	xmm1, xmm1, xmm2
 9048        vpxor	xmm1, xmm1, xmm3
 9049        vmovdqu	xmm7, OWORD PTR [rsp+96]
 9050        vmovdqu	xmm0, OWORD PTR [rcx+16]
 9051        vpshufd	xmm4, xmm7, 78
 9052        vpshufb	xmm0, xmm0, OWORD PTR L_avx1_aes_gcm_bswap_mask
 9053        vaesenc	xmm8, xmm8, [r15+32]
 9054        vpxor	xmm4, xmm4, xmm7
 9055        vpshufd	xmm5, xmm0, 78
 9056        vpxor	xmm5, xmm5, xmm0
 9057        vpclmulqdq	xmm6, xmm0, xmm7, 17
 9058        vaesenc	xmm9, xmm9, [r15+32]
 9059        vaesenc	xmm10, xmm10, [r15+32]
 9060        vpclmulqdq	xmm7, xmm0, xmm7, 0
 9061        vaesenc	xmm11, xmm11, [r15+32]
 9062        vaesenc	xmm12, xmm12, [r15+32]
 9063        vpclmulqdq	xmm4, xmm4, xmm5, 0
 9064        vaesenc	xmm13, xmm13, [r15+32]
 9065        vaesenc	xmm14, xmm14, [r15+32]
 9066        vaesenc	xmm15, xmm15, [r15+32]
 9067        vpxor	xmm1, xmm1, xmm7
 9068        vpxor	xmm2, xmm2, xmm7
 9069        vpxor	xmm1, xmm1, xmm6
 9070        vpxor	xmm3, xmm3, xmm6
 9071        vpxor	xmm1, xmm1, xmm4
 9072        vmovdqu	xmm7, OWORD PTR [rsp+80]
 9073        vmovdqu	xmm0, OWORD PTR [rcx+32]
 9074        vpshufd	xmm4, xmm7, 78
 9075        vpshufb	xmm0, xmm0, OWORD PTR L_avx1_aes_gcm_bswap_mask
 9076        vaesenc	xmm8, xmm8, [r15+48]
 9077        vpxor	xmm4, xmm4, xmm7
 9078        vpshufd	xmm5, xmm0, 78
 9079        vpxor	xmm5, xmm5, xmm0
 9080        vpclmulqdq	xmm6, xmm0, xmm7, 17
 9081        vaesenc	xmm9, xmm9, [r15+48]
 9082        vaesenc	xmm10, xmm10, [r15+48]
 9083        vpclmulqdq	xmm7, xmm0, xmm7, 0
 9084        vaesenc	xmm11, xmm11, [r15+48]
 9085        vaesenc	xmm12, xmm12, [r15+48]
 9086        vpclmulqdq	xmm4, xmm4, xmm5, 0
 9087        vaesenc	xmm13, xmm13, [r15+48]
 9088        vaesenc	xmm14, xmm14, [r15+48]
 9089        vaesenc	xmm15, xmm15, [r15+48]
 9090        vpxor	xmm1, xmm1, xmm7
 9091        vpxor	xmm2, xmm2, xmm7
 9092        vpxor	xmm1, xmm1, xmm6
 9093        vpxor	xmm3, xmm3, xmm6
 9094        vpxor	xmm1, xmm1, xmm4
 9095        vmovdqu	xmm7, OWORD PTR [rsp+64]
 9096        vmovdqu	xmm0, OWORD PTR [rcx+48]
 9097        vpshufd	xmm4, xmm7, 78
 9098        vpshufb	xmm0, xmm0, OWORD PTR L_avx1_aes_gcm_bswap_mask
 9099        vaesenc	xmm8, xmm8, [r15+64]
 9100        vpxor	xmm4, xmm4, xmm7
 9101        vpshufd	xmm5, xmm0, 78
 9102        vpxor	xmm5, xmm5, xmm0
 9103        vpclmulqdq	xmm6, xmm0, xmm7, 17
 9104        vaesenc	xmm9, xmm9, [r15+64]
 9105        vaesenc	xmm10, xmm10, [r15+64]
 9106        vpclmulqdq	xmm7, xmm0, xmm7, 0
 9107        vaesenc	xmm11, xmm11, [r15+64]
 9108        vaesenc	xmm12, xmm12, [r15+64]
 9109        vpclmulqdq	xmm4, xmm4, xmm5, 0
 9110        vaesenc	xmm13, xmm13, [r15+64]
 9111        vaesenc	xmm14, xmm14, [r15+64]
 9112        vaesenc	xmm15, xmm15, [r15+64]
 9113        vpxor	xmm1, xmm1, xmm7
 9114        vpxor	xmm2, xmm2, xmm7
 9115        vpxor	xmm1, xmm1, xmm6
 9116        vpxor	xmm3, xmm3, xmm6
 9117        vpxor	xmm1, xmm1, xmm4
 9118        vmovdqu	xmm7, OWORD PTR [rsp+48]
 9119        vmovdqu	xmm0, OWORD PTR [rcx+64]
 9120        vpshufd	xmm4, xmm7, 78
 9121        vpshufb	xmm0, xmm0, OWORD PTR L_avx1_aes_gcm_bswap_mask
 9122        vaesenc	xmm8, xmm8, [r15+80]
 9123        vpxor	xmm4, xmm4, xmm7
 9124        vpshufd	xmm5, xmm0, 78
 9125        vpxor	xmm5, xmm5, xmm0
 9126        vpclmulqdq	xmm6, xmm0, xmm7, 17
 9127        vaesenc	xmm9, xmm9, [r15+80]
 9128        vaesenc	xmm10, xmm10, [r15+80]
 9129        vpclmulqdq	xmm7, xmm0, xmm7, 0
 9130        vaesenc	xmm11, xmm11, [r15+80]
 9131        vaesenc	xmm12, xmm12, [r15+80]
 9132        vpclmulqdq	xmm4, xmm4, xmm5, 0
 9133        vaesenc	xmm13, xmm13, [r15+80]
 9134        vaesenc	xmm14, xmm14, [r15+80]
 9135        vaesenc	xmm15, xmm15, [r15+80]
 9136        vpxor	xmm1, xmm1, xmm7
 9137        vpxor	xmm2, xmm2, xmm7
 9138        vpxor	xmm1, xmm1, xmm6
 9139        vpxor	xmm3, xmm3, xmm6
 9140        vpxor	xmm1, xmm1, xmm4
 9141        vmovdqu	xmm7, OWORD PTR [rsp+32]
 9142        vmovdqu	xmm0, OWORD PTR [rcx+80]
 9143        vpshufd	xmm4, xmm7, 78
 9144        vpshufb	xmm0, xmm0, OWORD PTR L_avx1_aes_gcm_bswap_mask
 9145        vaesenc	xmm8, xmm8, [r15+96]
 9146        vpxor	xmm4, xmm4, xmm7
 9147        vpshufd	xmm5, xmm0, 78
 9148        vpxor	xmm5, xmm5, xmm0
 9149        vpclmulqdq	xmm6, xmm0, xmm7, 17
 9150        vaesenc	xmm9, xmm9, [r15+96]
 9151        vaesenc	xmm10, xmm10, [r15+96]
 9152        vpclmulqdq	xmm7, xmm0, xmm7, 0
 9153        vaesenc	xmm11, xmm11, [r15+96]
 9154        vaesenc	xmm12, xmm12, [r15+96]
 9155        vpclmulqdq	xmm4, xmm4, xmm5, 0
 9156        vaesenc	xmm13, xmm13, [r15+96]
 9157        vaesenc	xmm14, xmm14, [r15+96]
 9158        vaesenc	xmm15, xmm15, [r15+96]
 9159        vpxor	xmm1, xmm1, xmm7
 9160        vpxor	xmm2, xmm2, xmm7
 9161        vpxor	xmm1, xmm1, xmm6
 9162        vpxor	xmm3, xmm3, xmm6
 9163        vpxor	xmm1, xmm1, xmm4
 9164        vmovdqu	xmm7, OWORD PTR [rsp+16]
 9165        vmovdqu	xmm0, OWORD PTR [rcx+96]
 9166        vpshufd	xmm4, xmm7, 78
 9167        vpshufb	xmm0, xmm0, OWORD PTR L_avx1_aes_gcm_bswap_mask
 9168        vaesenc	xmm8, xmm8, [r15+112]
 9169        vpxor	xmm4, xmm4, xmm7
 9170        vpshufd	xmm5, xmm0, 78
 9171        vpxor	xmm5, xmm5, xmm0
 9172        vpclmulqdq	xmm6, xmm0, xmm7, 17
 9173        vaesenc	xmm9, xmm9, [r15+112]
 9174        vaesenc	xmm10, xmm10, [r15+112]
 9175        vpclmulqdq	xmm7, xmm0, xmm7, 0
 9176        vaesenc	xmm11, xmm11, [r15+112]
 9177        vaesenc	xmm12, xmm12, [r15+112]
 9178        vpclmulqdq	xmm4, xmm4, xmm5, 0
 9179        vaesenc	xmm13, xmm13, [r15+112]
 9180        vaesenc	xmm14, xmm14, [r15+112]
 9181        vaesenc	xmm15, xmm15, [r15+112]
 9182        vpxor	xmm1, xmm1, xmm7
 9183        vpxor	xmm2, xmm2, xmm7
 9184        vpxor	xmm1, xmm1, xmm6
 9185        vpxor	xmm3, xmm3, xmm6
 9186        vpxor	xmm1, xmm1, xmm4
 9187        vmovdqu	xmm7, OWORD PTR [rsp]
 9188        vmovdqu	xmm0, OWORD PTR [rcx+112]
 9189        vpshufd	xmm4, xmm7, 78
 9190        vpshufb	xmm0, xmm0, OWORD PTR L_avx1_aes_gcm_bswap_mask
 9191        vaesenc	xmm8, xmm8, [r15+128]
 9192        vpxor	xmm4, xmm4, xmm7
 9193        vpshufd	xmm5, xmm0, 78
 9194        vpxor	xmm5, xmm5, xmm0
 9195        vpclmulqdq	xmm6, xmm0, xmm7, 17
 9196        vaesenc	xmm9, xmm9, [r15+128]
 9197        vaesenc	xmm10, xmm10, [r15+128]
 9198        vpclmulqdq	xmm7, xmm0, xmm7, 0
 9199        vaesenc	xmm11, xmm11, [r15+128]
 9200        vaesenc	xmm12, xmm12, [r15+128]
 9201        vpclmulqdq	xmm4, xmm4, xmm5, 0
 9202        vaesenc	xmm13, xmm13, [r15+128]
 9203        vaesenc	xmm14, xmm14, [r15+128]
 9204        vaesenc	xmm15, xmm15, [r15+128]
 9205        vpxor	xmm1, xmm1, xmm7
 9206        vpxor	xmm2, xmm2, xmm7
 9207        vpxor	xmm1, xmm1, xmm6
 9208        vpxor	xmm3, xmm3, xmm6
 9209        vpxor	xmm1, xmm1, xmm4
 9210        vpslldq	xmm5, xmm1, 8
 9211        vpsrldq	xmm1, xmm1, 8
 9212        vaesenc	xmm8, xmm8, [r15+144]
 9213        vpxor	xmm2, xmm2, xmm5
 9214        vpxor	xmm3, xmm3, xmm1
 9215        vaesenc	xmm9, xmm9, [r15+144]
 9216        vpslld	xmm7, xmm2, 31
 9217        vpslld	xmm4, xmm2, 30
 9218        vpslld	xmm5, xmm2, 25
 9219        vaesenc	xmm10, xmm10, [r15+144]
 9220        vpxor	xmm7, xmm7, xmm4
 9221        vpxor	xmm7, xmm7, xmm5
 9222        vaesenc	xmm11, xmm11, [r15+144]
 9223        vpsrldq	xmm4, xmm7, 4
 9224        vpslldq	xmm7, xmm7, 12
 9225        vaesenc	xmm12, xmm12, [r15+144]
 9226        vpxor	xmm2, xmm2, xmm7
 9227        vpsrld	xmm5, xmm2, 1
 9228        vaesenc	xmm13, xmm13, [r15+144]
 9229        vpsrld	xmm1, xmm2, 2
 9230        vpsrld	xmm0, xmm2, 7
 9231        vaesenc	xmm14, xmm14, [r15+144]
 9232        vpxor	xmm5, xmm5, xmm1
 9233        vpxor	xmm5, xmm5, xmm0
 9234        vaesenc	xmm15, xmm15, [r15+144]
 9235        vpxor	xmm5, xmm5, xmm4
 9236        vpxor	xmm2, xmm2, xmm5
 9237        vpxor	xmm2, xmm2, xmm3
 9238        cmp	r10d, 11
 9239        vmovdqa	xmm7, OWORD PTR [r15+160]
 9240        jl	L_AES_GCM_decrypt_avx1_aesenc_128_ghash_avx_done
 9241        vaesenc	xmm8, xmm8, xmm7
 9242        vaesenc	xmm9, xmm9, xmm7
 9243        vaesenc	xmm10, xmm10, xmm7
 9244        vaesenc	xmm11, xmm11, xmm7
 9245        vaesenc	xmm12, xmm12, xmm7
 9246        vaesenc	xmm13, xmm13, xmm7
 9247        vaesenc	xmm14, xmm14, xmm7
 9248        vaesenc	xmm15, xmm15, xmm7
 9249        vmovdqa	xmm7, OWORD PTR [r15+176]
 9250        vaesenc	xmm8, xmm8, xmm7
 9251        vaesenc	xmm9, xmm9, xmm7
 9252        vaesenc	xmm10, xmm10, xmm7
 9253        vaesenc	xmm11, xmm11, xmm7
 9254        vaesenc	xmm12, xmm12, xmm7
 9255        vaesenc	xmm13, xmm13, xmm7
 9256        vaesenc	xmm14, xmm14, xmm7
 9257        vaesenc	xmm15, xmm15, xmm7
 9258        cmp	r10d, 13
 9259        vmovdqa	xmm7, OWORD PTR [r15+192]
 9260        jl	L_AES_GCM_decrypt_avx1_aesenc_128_ghash_avx_done
 9261        vaesenc	xmm8, xmm8, xmm7
 9262        vaesenc	xmm9, xmm9, xmm7
 9263        vaesenc	xmm10, xmm10, xmm7
 9264        vaesenc	xmm11, xmm11, xmm7
 9265        vaesenc	xmm12, xmm12, xmm7
 9266        vaesenc	xmm13, xmm13, xmm7
 9267        vaesenc	xmm14, xmm14, xmm7
 9268        vaesenc	xmm15, xmm15, xmm7
 9269        vmovdqa	xmm7, OWORD PTR [r15+208]
 9270        vaesenc	xmm8, xmm8, xmm7
 9271        vaesenc	xmm9, xmm9, xmm7
 9272        vaesenc	xmm10, xmm10, xmm7
 9273        vaesenc	xmm11, xmm11, xmm7
 9274        vaesenc	xmm12, xmm12, xmm7
 9275        vaesenc	xmm13, xmm13, xmm7
 9276        vaesenc	xmm14, xmm14, xmm7
 9277        vaesenc	xmm15, xmm15, xmm7
 9278        vmovdqa	xmm7, OWORD PTR [r15+224]
 9279L_AES_GCM_decrypt_avx1_aesenc_128_ghash_avx_done:
 9280        vaesenclast	xmm8, xmm8, xmm7
 9281        vaesenclast	xmm9, xmm9, xmm7
 9282        vmovdqu	xmm0, OWORD PTR [rcx]
 9283        vmovdqu	xmm1, OWORD PTR [rcx+16]
 9284        vpxor	xmm8, xmm8, xmm0
 9285        vpxor	xmm9, xmm9, xmm1
 9286        vmovdqu	OWORD PTR [rdx], xmm8
 9287        vmovdqu	OWORD PTR [rdx+16], xmm9
 9288        vaesenclast	xmm10, xmm10, xmm7
 9289        vaesenclast	xmm11, xmm11, xmm7
 9290        vmovdqu	xmm0, OWORD PTR [rcx+32]
 9291        vmovdqu	xmm1, OWORD PTR [rcx+48]
 9292        vpxor	xmm10, xmm10, xmm0
 9293        vpxor	xmm11, xmm11, xmm1
 9294        vmovdqu	OWORD PTR [rdx+32], xmm10
 9295        vmovdqu	OWORD PTR [rdx+48], xmm11
 9296        vaesenclast	xmm12, xmm12, xmm7
 9297        vaesenclast	xmm13, xmm13, xmm7
 9298        vmovdqu	xmm0, OWORD PTR [rcx+64]
 9299        vmovdqu	xmm1, OWORD PTR [rcx+80]
 9300        vpxor	xmm12, xmm12, xmm0
 9301        vpxor	xmm13, xmm13, xmm1
 9302        vmovdqu	OWORD PTR [rdx+64], xmm12
 9303        vmovdqu	OWORD PTR [rdx+80], xmm13
 9304        vaesenclast	xmm14, xmm14, xmm7
 9305        vaesenclast	xmm15, xmm15, xmm7
 9306        vmovdqu	xmm0, OWORD PTR [rcx+96]
 9307        vmovdqu	xmm1, OWORD PTR [rcx+112]
 9308        vpxor	xmm14, xmm14, xmm0
 9309        vpxor	xmm15, xmm15, xmm1
 9310        vmovdqu	OWORD PTR [rdx+96], xmm14
 9311        vmovdqu	OWORD PTR [rdx+112], xmm15
 9312        add	ebx, 128
 9313        cmp	ebx, r13d
 9314        jl	L_AES_GCM_decrypt_avx1_ghash_128
 9315        vmovdqa	xmm6, xmm2
 9316        vmovdqu	xmm5, OWORD PTR [rsp]
 9317L_AES_GCM_decrypt_avx1_done_128:
 9318        mov	edx, r9d
 9319        cmp	ebx, edx
 9320        jge	L_AES_GCM_decrypt_avx1_done_dec
 9321        mov	r13d, r9d
 9322        and	r13d, 4294967280
 9323        cmp	ebx, r13d
 9324        jge	L_AES_GCM_decrypt_avx1_last_block_done
 9325L_AES_GCM_decrypt_avx1_last_block_start:
 9326        vmovdqu	xmm13, OWORD PTR [rdi+rbx]
 9327        vmovdqa	xmm0, xmm5
 9328        vpshufb	xmm1, xmm13, OWORD PTR L_avx1_aes_gcm_bswap_mask
 9329        vpxor	xmm1, xmm1, xmm6
 9330        vmovdqu	xmm9, OWORD PTR [rsp+128]
 9331        vpshufb	xmm8, xmm9, OWORD PTR L_avx1_aes_gcm_bswap_epi64
 9332        vpaddd	xmm9, xmm9, OWORD PTR L_avx1_aes_gcm_one
 9333        vmovdqu	OWORD PTR [rsp+128], xmm9
 9334        vpxor	xmm8, xmm8, [r15]
 9335        vpclmulqdq	xmm10, xmm1, xmm0, 16
 9336        vaesenc	xmm8, xmm8, [r15+16]
 9337        vaesenc	xmm8, xmm8, [r15+32]
 9338        vpclmulqdq	xmm11, xmm1, xmm0, 1
 9339        vaesenc	xmm8, xmm8, [r15+48]
 9340        vaesenc	xmm8, xmm8, [r15+64]
 9341        vpclmulqdq	xmm12, xmm1, xmm0, 0
 9342        vaesenc	xmm8, xmm8, [r15+80]
 9343        vpclmulqdq	xmm1, xmm1, xmm0, 17
 9344        vaesenc	xmm8, xmm8, [r15+96]
 9345        vpxor	xmm10, xmm10, xmm11
 9346        vpslldq	xmm2, xmm10, 8
 9347        vpsrldq	xmm10, xmm10, 8
 9348        vaesenc	xmm8, xmm8, [r15+112]
 9349        vpxor	xmm2, xmm2, xmm12
 9350        vpxor	xmm3, xmm1, xmm10
 9351        vmovdqa	xmm0, OWORD PTR L_avx1_aes_gcm_mod2_128
 9352        vpclmulqdq	xmm11, xmm2, xmm0, 16
 9353        vaesenc	xmm8, xmm8, [r15+128]
 9354        vpshufd	xmm10, xmm2, 78
 9355        vpxor	xmm10, xmm10, xmm11
 9356        vpclmulqdq	xmm11, xmm10, xmm0, 16
 9357        vaesenc	xmm8, xmm8, [r15+144]
 9358        vpshufd	xmm10, xmm10, 78
 9359        vpxor	xmm10, xmm10, xmm11
 9360        vpxor	xmm6, xmm10, xmm3
 9361        cmp	r10d, 11
 9362        vmovdqa	xmm9, OWORD PTR [r15+160]
 9363        jl	L_AES_GCM_decrypt_avx1_aesenc_gfmul_last
 9364        vaesenc	xmm8, xmm8, xmm9
 9365        vaesenc	xmm8, xmm8, [r15+176]
 9366        cmp	r10d, 13
 9367        vmovdqa	xmm9, OWORD PTR [r15+192]
 9368        jl	L_AES_GCM_decrypt_avx1_aesenc_gfmul_last
 9369        vaesenc	xmm8, xmm8, xmm9
 9370        vaesenc	xmm8, xmm8, [r15+208]
 9371        vmovdqa	xmm9, OWORD PTR [r15+224]
 9372L_AES_GCM_decrypt_avx1_aesenc_gfmul_last:
 9373        vaesenclast	xmm8, xmm8, xmm9
 9374        vmovdqa	xmm0, xmm13
 9375        vpxor	xmm8, xmm8, xmm0
 9376        vmovdqu	OWORD PTR [rsi+rbx], xmm8
 9377        add	ebx, 16
 9378        cmp	ebx, r13d
 9379        jl	L_AES_GCM_decrypt_avx1_last_block_start
 9380L_AES_GCM_decrypt_avx1_last_block_done:
 9381        mov	ecx, r9d
 9382        mov	edx, ecx
 9383        and	ecx, 15
 9384        jz	L_AES_GCM_decrypt_avx1_aesenc_last15_dec_avx_done
 9385        vmovdqu	xmm4, OWORD PTR [rsp+128]
 9386        vpshufb	xmm4, xmm4, OWORD PTR L_avx1_aes_gcm_bswap_epi64
 9387        vpxor	xmm4, xmm4, [r15]
 9388        vaesenc	xmm4, xmm4, [r15+16]
 9389        vaesenc	xmm4, xmm4, [r15+32]
 9390        vaesenc	xmm4, xmm4, [r15+48]
 9391        vaesenc	xmm4, xmm4, [r15+64]
 9392        vaesenc	xmm4, xmm4, [r15+80]
 9393        vaesenc	xmm4, xmm4, [r15+96]
 9394        vaesenc	xmm4, xmm4, [r15+112]
 9395        vaesenc	xmm4, xmm4, [r15+128]
 9396        vaesenc	xmm4, xmm4, [r15+144]
 9397        cmp	r10d, 11
 9398        vmovdqa	xmm9, OWORD PTR [r15+160]
 9399        jl	L_AES_GCM_decrypt_avx1_aesenc_last15_dec_avx_aesenc_avx_last
 9400        vaesenc	xmm4, xmm4, xmm9
 9401        vaesenc	xmm4, xmm4, [r15+176]
 9402        cmp	r10d, 13
 9403        vmovdqa	xmm9, OWORD PTR [r15+192]
 9404        jl	L_AES_GCM_decrypt_avx1_aesenc_last15_dec_avx_aesenc_avx_last
 9405        vaesenc	xmm4, xmm4, xmm9
 9406        vaesenc	xmm4, xmm4, [r15+208]
 9407        vmovdqa	xmm9, OWORD PTR [r15+224]
 9408L_AES_GCM_decrypt_avx1_aesenc_last15_dec_avx_aesenc_avx_last:
 9409        vaesenclast	xmm4, xmm4, xmm9
 9410        sub	rsp, 32
 9411        xor	ecx, ecx
 9412        vmovdqu	OWORD PTR [rsp], xmm4
 9413        vpxor	xmm0, xmm0, xmm0
 9414        vmovdqu	OWORD PTR [rsp+16], xmm0
 9415L_AES_GCM_decrypt_avx1_aesenc_last15_dec_avx_loop:
 9416        movzx	r13d, BYTE PTR [rdi+rbx]
 9417        mov	BYTE PTR [rsp+rcx+16], r13b
 9418        xor	r13b, BYTE PTR [rsp+rcx]
 9419        mov	BYTE PTR [rsi+rbx], r13b
 9420        inc	ebx
 9421        inc	ecx
 9422        cmp	ebx, edx
 9423        jl	L_AES_GCM_decrypt_avx1_aesenc_last15_dec_avx_loop
 9424        vmovdqu	xmm4, OWORD PTR [rsp+16]
 9425        add	rsp, 32
 9426        vpshufb	xmm4, xmm4, OWORD PTR L_avx1_aes_gcm_bswap_mask
 9427        vpxor	xmm6, xmm6, xmm4
 9428        ; ghash_gfmul_red_avx
 9429        vpshufd	xmm9, xmm5, 78
 9430        vpshufd	xmm10, xmm6, 78
 9431        vpclmulqdq	xmm11, xmm6, xmm5, 17
 9432        vpclmulqdq	xmm8, xmm6, xmm5, 0
 9433        vpxor	xmm9, xmm9, xmm5
 9434        vpxor	xmm10, xmm10, xmm6
 9435        vpclmulqdq	xmm9, xmm9, xmm10, 0
 9436        vpxor	xmm9, xmm9, xmm8
 9437        vpxor	xmm9, xmm9, xmm11
 9438        vpslldq	xmm10, xmm9, 8
 9439        vpsrldq	xmm9, xmm9, 8
 9440        vpxor	xmm8, xmm8, xmm10
 9441        vpxor	xmm6, xmm11, xmm9
 9442        vpslld	xmm12, xmm8, 31
 9443        vpslld	xmm13, xmm8, 30
 9444        vpslld	xmm14, xmm8, 25
 9445        vpxor	xmm12, xmm12, xmm13
 9446        vpxor	xmm12, xmm12, xmm14
 9447        vpsrldq	xmm13, xmm12, 4
 9448        vpslldq	xmm12, xmm12, 12
 9449        vpxor	xmm8, xmm8, xmm12
 9450        vpsrld	xmm14, xmm8, 1
 9451        vpsrld	xmm10, xmm8, 2
 9452        vpsrld	xmm9, xmm8, 7
 9453        vpxor	xmm14, xmm14, xmm10
 9454        vpxor	xmm14, xmm14, xmm9
 9455        vpxor	xmm14, xmm14, xmm13
 9456        vpxor	xmm14, xmm14, xmm8
 9457        vpxor	xmm6, xmm6, xmm14
 9458L_AES_GCM_decrypt_avx1_aesenc_last15_dec_avx_done:
 9459L_AES_GCM_decrypt_avx1_done_dec:
 9460        mov	edx, r9d
 9461        mov	ecx, r11d
 9462        shl	rdx, 3
 9463        shl	rcx, 3
 9464        vmovq	xmm0, rdx
 9465        vmovq	xmm1, rcx
 9466        vpunpcklqdq	xmm0, xmm0, xmm1
 9467        vpxor	xmm6, xmm6, xmm0
 9468        ; ghash_gfmul_red_avx
 9469        vpshufd	xmm9, xmm5, 78
 9470        vpshufd	xmm10, xmm6, 78
 9471        vpclmulqdq	xmm11, xmm6, xmm5, 17
 9472        vpclmulqdq	xmm8, xmm6, xmm5, 0
 9473        vpxor	xmm9, xmm9, xmm5
 9474        vpxor	xmm10, xmm10, xmm6
 9475        vpclmulqdq	xmm9, xmm9, xmm10, 0
 9476        vpxor	xmm9, xmm9, xmm8
 9477        vpxor	xmm9, xmm9, xmm11
 9478        vpslldq	xmm10, xmm9, 8
 9479        vpsrldq	xmm9, xmm9, 8
 9480        vpxor	xmm8, xmm8, xmm10
 9481        vpxor	xmm6, xmm11, xmm9
 9482        vpslld	xmm12, xmm8, 31
 9483        vpslld	xmm13, xmm8, 30
 9484        vpslld	xmm14, xmm8, 25
 9485        vpxor	xmm12, xmm12, xmm13
 9486        vpxor	xmm12, xmm12, xmm14
 9487        vpsrldq	xmm13, xmm12, 4
 9488        vpslldq	xmm12, xmm12, 12
 9489        vpxor	xmm8, xmm8, xmm12
 9490        vpsrld	xmm14, xmm8, 1
 9491        vpsrld	xmm10, xmm8, 2
 9492        vpsrld	xmm9, xmm8, 7
 9493        vpxor	xmm14, xmm14, xmm10
 9494        vpxor	xmm14, xmm14, xmm9
 9495        vpxor	xmm14, xmm14, xmm13
 9496        vpxor	xmm14, xmm14, xmm8
 9497        vpxor	xmm6, xmm6, xmm14
 9498        vpshufb	xmm6, xmm6, OWORD PTR L_avx1_aes_gcm_bswap_mask
 9499        vmovdqu	xmm0, OWORD PTR [rsp+144]
 9500        vpxor	xmm0, xmm0, xmm6
 9501        cmp	r14d, 16
 9502        je	L_AES_GCM_decrypt_avx1_cmp_tag_16
 9503        sub	rsp, 16
 9504        xor	rcx, rcx
 9505        xor	rbx, rbx
 9506        vmovdqu	OWORD PTR [rsp], xmm0
 9507L_AES_GCM_decrypt_avx1_cmp_tag_loop:
 9508        movzx	r13d, BYTE PTR [rsp+rcx]
 9509        xor	r13b, BYTE PTR [r8+rcx]
 9510        or	bl, r13b
 9511        inc	ecx
 9512        cmp	ecx, r14d
 9513        jne	L_AES_GCM_decrypt_avx1_cmp_tag_loop
 9514        cmp	bl, 0
 9515        sete	bl
 9516        add	rsp, 16
 9517        xor	rcx, rcx
 9518        jmp	L_AES_GCM_decrypt_avx1_cmp_tag_done
 9519L_AES_GCM_decrypt_avx1_cmp_tag_16:
 9520        vmovdqu	xmm1, OWORD PTR [r8]
 9521        vpcmpeqb	xmm0, xmm0, xmm1
 9522        vpmovmskb	rdx, xmm0
 9523        ; %%edx == 0xFFFF then return 1 else => return 0
 9524        xor	ebx, ebx
 9525        cmp	edx, 65535
 9526        sete	bl
 9527L_AES_GCM_decrypt_avx1_cmp_tag_done:
 9528        mov	DWORD PTR [rbp], ebx
 9529        vzeroupper
 9530        vmovdqu	xmm6, OWORD PTR [rsp+168]
 9531        vmovdqu	xmm7, OWORD PTR [rsp+184]
 9532        vmovdqu	xmm8, OWORD PTR [rsp+200]
 9533        vmovdqu	xmm9, OWORD PTR [rsp+216]
 9534        vmovdqu	xmm10, OWORD PTR [rsp+232]
 9535        vmovdqu	xmm11, OWORD PTR [rsp+248]
 9536        vmovdqu	xmm12, OWORD PTR [rsp+264]
 9537        vmovdqu	xmm13, OWORD PTR [rsp+280]
 9538        vmovdqu	xmm14, OWORD PTR [rsp+296]
 9539        vmovdqu	xmm15, OWORD PTR [rsp+312]
 9540        add	rsp, 328
 9541        pop	rbp
 9542        pop	r15
 9543        pop	r14
 9544        pop	rbx
 9545        pop	r12
 9546        pop	rsi
 9547        pop	rdi
 9548        pop	r13
 9549        ret
 9550AES_GCM_decrypt_avx1 ENDP
 9551_TEXT ENDS
 9552_TEXT SEGMENT READONLY PARA
 9553AES_GCM_init_avx1 PROC
 9554        push	rdi
 9555        push	rsi
 9556        push	r12
 9557        push	r13
 9558        mov	rdi, rcx
 9559        mov	rsi, rdx
 9560        mov	r10, r8
 9561        mov	r11d, r9d
 9562        mov	rax, QWORD PTR [rsp+72]
 9563        mov	r8, QWORD PTR [rsp+80]
 9564        mov	r9, QWORD PTR [rsp+88]
 9565        sub	rsp, 80
 9566        vmovdqu	OWORD PTR [rsp+16], xmm6
 9567        vmovdqu	OWORD PTR [rsp+32], xmm7
 9568        vmovdqu	OWORD PTR [rsp+48], xmm8
 9569        vmovdqu	OWORD PTR [rsp+64], xmm15
 9570        vpxor	xmm4, xmm4, xmm4
 9571        mov	edx, r11d
 9572        cmp	edx, 12
 9573        jne	L_AES_GCM_init_avx1_iv_not_12
 9574        ; # Calculate values when IV is 12 bytes
 9575        ; Set counter based on IV
 9576        mov	ecx, 16777216
 9577        vmovq	xmm4, QWORD PTR [r10]
 9578        vpinsrd	xmm4, xmm4, DWORD PTR [r10+8], 2
 9579        vpinsrd	xmm4, xmm4, ecx, 3
 9580        ; H = Encrypt X(=0) and T = Encrypt counter
 9581        vmovdqa	xmm5, OWORD PTR [rdi]
 9582        vpxor	xmm1, xmm4, xmm5
 9583        vmovdqa	xmm6, OWORD PTR [rdi+16]
 9584        vaesenc	xmm5, xmm5, xmm6
 9585        vaesenc	xmm1, xmm1, xmm6
 9586        vmovdqa	xmm6, OWORD PTR [rdi+32]
 9587        vaesenc	xmm5, xmm5, xmm6
 9588        vaesenc	xmm1, xmm1, xmm6
 9589        vmovdqa	xmm6, OWORD PTR [rdi+48]
 9590        vaesenc	xmm5, xmm5, xmm6
 9591        vaesenc	xmm1, xmm1, xmm6
 9592        vmovdqa	xmm6, OWORD PTR [rdi+64]
 9593        vaesenc	xmm5, xmm5, xmm6
 9594        vaesenc	xmm1, xmm1, xmm6
 9595        vmovdqa	xmm6, OWORD PTR [rdi+80]
 9596        vaesenc	xmm5, xmm5, xmm6
 9597        vaesenc	xmm1, xmm1, xmm6
 9598        vmovdqa	xmm6, OWORD PTR [rdi+96]
 9599        vaesenc	xmm5, xmm5, xmm6
 9600        vaesenc	xmm1, xmm1, xmm6
 9601        vmovdqa	xmm6, OWORD PTR [rdi+112]
 9602        vaesenc	xmm5, xmm5, xmm6
 9603        vaesenc	xmm1, xmm1, xmm6
 9604        vmovdqa	xmm6, OWORD PTR [rdi+128]
 9605        vaesenc	xmm5, xmm5, xmm6
 9606        vaesenc	xmm1, xmm1, xmm6
 9607        vmovdqa	xmm6, OWORD PTR [rdi+144]
 9608        vaesenc	xmm5, xmm5, xmm6
 9609        vaesenc	xmm1, xmm1, xmm6
 9610        cmp	esi, 11
 9611        vmovdqa	xmm6, OWORD PTR [rdi+160]
 9612        jl	L_AES_GCM_init_avx1_calc_iv_12_last
 9613        vaesenc	xmm5, xmm5, xmm6
 9614        vaesenc	xmm1, xmm1, xmm6
 9615        vmovdqa	xmm6, OWORD PTR [rdi+176]
 9616        vaesenc	xmm5, xmm5, xmm6
 9617        vaesenc	xmm1, xmm1, xmm6
 9618        cmp	esi, 13
 9619        vmovdqa	xmm6, OWORD PTR [rdi+192]
 9620        jl	L_AES_GCM_init_avx1_calc_iv_12_last
 9621        vaesenc	xmm5, xmm5, xmm6
 9622        vaesenc	xmm1, xmm1, xmm6
 9623        vmovdqa	xmm6, OWORD PTR [rdi+208]
 9624        vaesenc	xmm5, xmm5, xmm6
 9625        vaesenc	xmm1, xmm1, xmm6
 9626        vmovdqa	xmm6, OWORD PTR [rdi+224]
 9627L_AES_GCM_init_avx1_calc_iv_12_last:
 9628        vaesenclast	xmm5, xmm5, xmm6
 9629        vaesenclast	xmm1, xmm1, xmm6
 9630        vpshufb	xmm5, xmm5, OWORD PTR L_avx1_aes_gcm_bswap_mask
 9631        vmovdqu	xmm15, xmm1
 9632        jmp	L_AES_GCM_init_avx1_iv_done
 9633L_AES_GCM_init_avx1_iv_not_12:
 9634        ; Calculate values when IV is not 12 bytes
 9635        ; H = Encrypt X(=0)
 9636        vmovdqa	xmm5, OWORD PTR [rdi]
 9637        vaesenc	xmm5, xmm5, [rdi+16]
 9638        vaesenc	xmm5, xmm5, [rdi+32]
 9639        vaesenc	xmm5, xmm5, [rdi+48]
 9640        vaesenc	xmm5, xmm5, [rdi+64]
 9641        vaesenc	xmm5, xmm5, [rdi+80]
 9642        vaesenc	xmm5, xmm5, [rdi+96]
 9643        vaesenc	xmm5, xmm5, [rdi+112]
 9644        vaesenc	xmm5, xmm5, [rdi+128]
 9645        vaesenc	xmm5, xmm5, [rdi+144]
 9646        cmp	esi, 11
 9647        vmovdqa	xmm8, OWORD PTR [rdi+160]
 9648        jl	L_AES_GCM_init_avx1_calc_iv_1_aesenc_avx_last
 9649        vaesenc	xmm5, xmm5, xmm8
 9650        vaesenc	xmm5, xmm5, [rdi+176]
 9651        cmp	esi, 13
 9652        vmovdqa	xmm8, OWORD PTR [rdi+192]
 9653        jl	L_AES_GCM_init_avx1_calc_iv_1_aesenc_avx_last
 9654        vaesenc	xmm5, xmm5, xmm8
 9655        vaesenc	xmm5, xmm5, [rdi+208]
 9656        vmovdqa	xmm8, OWORD PTR [rdi+224]
 9657L_AES_GCM_init_avx1_calc_iv_1_aesenc_avx_last:
 9658        vaesenclast	xmm5, xmm5, xmm8
 9659        vpshufb	xmm5, xmm5, OWORD PTR L_avx1_aes_gcm_bswap_mask
 9660        ; Calc counter
 9661        ; Initialization vector
 9662        cmp	edx, 0
 9663        mov	rcx, 0
 9664        je	L_AES_GCM_init_avx1_calc_iv_done
 9665        cmp	edx, 16
 9666        jl	L_AES_GCM_init_avx1_calc_iv_lt16
 9667        and	edx, 4294967280
 9668L_AES_GCM_init_avx1_calc_iv_16_loop:
 9669        vmovdqu	xmm7, OWORD PTR [r10+rcx]
 9670        vpshufb	xmm7, xmm7, OWORD PTR L_avx1_aes_gcm_bswap_mask
 9671        vpxor	xmm4, xmm4, xmm7
 9672        ; ghash_gfmul_avx
 9673        vpshufd	xmm1, xmm4, 78
 9674        vpshufd	xmm2, xmm5, 78
 9675        vpclmulqdq	xmm3, xmm5, xmm4, 17
 9676        vpclmulqdq	xmm0, xmm5, xmm4, 0
 9677        vpxor	xmm1, xmm1, xmm4
 9678        vpxor	xmm2, xmm2, xmm5
 9679        vpclmulqdq	xmm1, xmm1, xmm2, 0
 9680        vpxor	xmm1, xmm1, xmm0
 9681        vpxor	xmm1, xmm1, xmm3
 9682        vmovdqa	xmm6, xmm0
 9683        vmovdqa	xmm4, xmm3
 9684        vpslldq	xmm2, xmm1, 8
 9685        vpsrldq	xmm1, xmm1, 8
 9686        vpxor	xmm6, xmm6, xmm2
 9687        vpxor	xmm4, xmm4, xmm1
 9688        vpsrld	xmm0, xmm6, 31
 9689        vpsrld	xmm1, xmm4, 31
 9690        vpslld	xmm6, xmm6, 1
 9691        vpslld	xmm4, xmm4, 1
 9692        vpsrldq	xmm2, xmm0, 12
 9693        vpslldq	xmm0, xmm0, 4
 9694        vpslldq	xmm1, xmm1, 4
 9695        vpor	xmm4, xmm4, xmm2
 9696        vpor	xmm6, xmm6, xmm0
 9697        vpor	xmm4, xmm4, xmm1
 9698        vpslld	xmm0, xmm6, 31
 9699        vpslld	xmm1, xmm6, 30
 9700        vpslld	xmm2, xmm6, 25
 9701        vpxor	xmm0, xmm0, xmm1
 9702        vpxor	xmm0, xmm0, xmm2
 9703        vmovdqa	xmm1, xmm0
 9704        vpsrldq	xmm1, xmm1, 4
 9705        vpslldq	xmm0, xmm0, 12
 9706        vpxor	xmm6, xmm6, xmm0
 9707        vpsrld	xmm2, xmm6, 1
 9708        vpsrld	xmm3, xmm6, 2
 9709        vpsrld	xmm0, xmm6, 7
 9710        vpxor	xmm2, xmm2, xmm3
 9711        vpxor	xmm2, xmm2, xmm0
 9712        vpxor	xmm2, xmm2, xmm1
 9713        vpxor	xmm2, xmm2, xmm6
 9714        vpxor	xmm4, xmm4, xmm2
 9715        add	ecx, 16
 9716        cmp	ecx, edx
 9717        jl	L_AES_GCM_init_avx1_calc_iv_16_loop
 9718        mov	edx, r11d
 9719        cmp	ecx, edx
 9720        je	L_AES_GCM_init_avx1_calc_iv_done
 9721L_AES_GCM_init_avx1_calc_iv_lt16:
 9722        sub	rsp, 16
 9723        vpxor	xmm7, xmm7, xmm7
 9724        xor	r13d, r13d
 9725        vmovdqu	OWORD PTR [rsp], xmm7
 9726L_AES_GCM_init_avx1_calc_iv_loop:
 9727        movzx	r12d, BYTE PTR [r10+rcx]
 9728        mov	BYTE PTR [rsp+r13], r12b
 9729        inc	ecx
 9730        inc	r13d
 9731        cmp	ecx, edx
 9732        jl	L_AES_GCM_init_avx1_calc_iv_loop
 9733        vmovdqu	xmm7, OWORD PTR [rsp]
 9734        add	rsp, 16
 9735        vpshufb	xmm7, xmm7, OWORD PTR L_avx1_aes_gcm_bswap_mask
 9736        vpxor	xmm4, xmm4, xmm7
 9737        ; ghash_gfmul_avx
 9738        vpshufd	xmm1, xmm4, 78
 9739        vpshufd	xmm2, xmm5, 78
 9740        vpclmulqdq	xmm3, xmm5, xmm4, 17
 9741        vpclmulqdq	xmm0, xmm5, xmm4, 0
 9742        vpxor	xmm1, xmm1, xmm4
 9743        vpxor	xmm2, xmm2, xmm5
 9744        vpclmulqdq	xmm1, xmm1, xmm2, 0
 9745        vpxor	xmm1, xmm1, xmm0
 9746        vpxor	xmm1, xmm1, xmm3
 9747        vmovdqa	xmm6, xmm0
 9748        vmovdqa	xmm4, xmm3
 9749        vpslldq	xmm2, xmm1, 8
 9750        vpsrldq	xmm1, xmm1, 8
 9751        vpxor	xmm6, xmm6, xmm2
 9752        vpxor	xmm4, xmm4, xmm1
 9753        vpsrld	xmm0, xmm6, 31
 9754        vpsrld	xmm1, xmm4, 31
 9755        vpslld	xmm6, xmm6, 1
 9756        vpslld	xmm4, xmm4, 1
 9757        vpsrldq	xmm2, xmm0, 12
 9758        vpslldq	xmm0, xmm0, 4
 9759        vpslldq	xmm1, xmm1, 4
 9760        vpor	xmm4, xmm4, xmm2
 9761        vpor	xmm6, xmm6, xmm0
 9762        vpor	xmm4, xmm4, xmm1
 9763        vpslld	xmm0, xmm6, 31
 9764        vpslld	xmm1, xmm6, 30
 9765        vpslld	xmm2, xmm6, 25
 9766        vpxor	xmm0, xmm0, xmm1
 9767        vpxor	xmm0, xmm0, xmm2
 9768        vmovdqa	xmm1, xmm0
 9769        vpsrldq	xmm1, xmm1, 4
 9770        vpslldq	xmm0, xmm0, 12
 9771        vpxor	xmm6, xmm6, xmm0
 9772        vpsrld	xmm2, xmm6, 1
 9773        vpsrld	xmm3, xmm6, 2
 9774        vpsrld	xmm0, xmm6, 7
 9775        vpxor	xmm2, xmm2, xmm3
 9776        vpxor	xmm2, xmm2, xmm0
 9777        vpxor	xmm2, xmm2, xmm1
 9778        vpxor	xmm2, xmm2, xmm6
 9779        vpxor	xmm4, xmm4, xmm2
 9780L_AES_GCM_init_avx1_calc_iv_done:
 9781        ; T = Encrypt counter
 9782        vpxor	xmm0, xmm0, xmm0
 9783        shl	edx, 3
 9784        vmovq	xmm0, rdx
 9785        vpxor	xmm4, xmm4, xmm0
 9786        ; ghash_gfmul_avx
 9787        vpshufd	xmm1, xmm4, 78
 9788        vpshufd	xmm2, xmm5, 78
 9789        vpclmulqdq	xmm3, xmm5, xmm4, 17
 9790        vpclmulqdq	xmm0, xmm5, xmm4, 0
 9791        vpxor	xmm1, xmm1, xmm4
 9792        vpxor	xmm2, xmm2, xmm5
 9793        vpclmulqdq	xmm1, xmm1, xmm2, 0
 9794        vpxor	xmm1, xmm1, xmm0
 9795        vpxor	xmm1, xmm1, xmm3
 9796        vmovdqa	xmm6, xmm0
 9797        vmovdqa	xmm4, xmm3
 9798        vpslldq	xmm2, xmm1, 8
 9799        vpsrldq	xmm1, xmm1, 8
 9800        vpxor	xmm6, xmm6, xmm2
 9801        vpxor	xmm4, xmm4, xmm1
 9802        vpsrld	xmm0, xmm6, 31
 9803        vpsrld	xmm1, xmm4, 31
 9804        vpslld	xmm6, xmm6, 1
 9805        vpslld	xmm4, xmm4, 1
 9806        vpsrldq	xmm2, xmm0, 12
 9807        vpslldq	xmm0, xmm0, 4
 9808        vpslldq	xmm1, xmm1, 4
 9809        vpor	xmm4, xmm4, xmm2
 9810        vpor	xmm6, xmm6, xmm0
 9811        vpor	xmm4, xmm4, xmm1
 9812        vpslld	xmm0, xmm6, 31
 9813        vpslld	xmm1, xmm6, 30
 9814        vpslld	xmm2, xmm6, 25
 9815        vpxor	xmm0, xmm0, xmm1
 9816        vpxor	xmm0, xmm0, xmm2
 9817        vmovdqa	xmm1, xmm0
 9818        vpsrldq	xmm1, xmm1, 4
 9819        vpslldq	xmm0, xmm0, 12
 9820        vpxor	xmm6, xmm6, xmm0
 9821        vpsrld	xmm2, xmm6, 1
 9822        vpsrld	xmm3, xmm6, 2
 9823        vpsrld	xmm0, xmm6, 7
 9824        vpxor	xmm2, xmm2, xmm3
 9825        vpxor	xmm2, xmm2, xmm0
 9826        vpxor	xmm2, xmm2, xmm1
 9827        vpxor	xmm2, xmm2, xmm6
 9828        vpxor	xmm4, xmm4, xmm2
 9829        vpshufb	xmm4, xmm4, OWORD PTR L_avx1_aes_gcm_bswap_mask
 9830        ;   Encrypt counter
 9831        vmovdqa	xmm7, OWORD PTR [rdi]
 9832        vpxor	xmm7, xmm7, xmm4
 9833        vaesenc	xmm7, xmm7, [rdi+16]
 9834        vaesenc	xmm7, xmm7, [rdi+32]
 9835        vaesenc	xmm7, xmm7, [rdi+48]
 9836        vaesenc	xmm7, xmm7, [rdi+64]
 9837        vaesenc	xmm7, xmm7, [rdi+80]
 9838        vaesenc	xmm7, xmm7, [rdi+96]
 9839        vaesenc	xmm7, xmm7, [rdi+112]
 9840        vaesenc	xmm7, xmm7, [rdi+128]
 9841        vaesenc	xmm7, xmm7, [rdi+144]
 9842        cmp	esi, 11
 9843        vmovdqa	xmm8, OWORD PTR [rdi+160]
 9844        jl	L_AES_GCM_init_avx1_calc_iv_2_aesenc_avx_last
 9845        vaesenc	xmm7, xmm7, xmm8
 9846        vaesenc	xmm7, xmm7, [rdi+176]
 9847        cmp	esi, 13
 9848        vmovdqa	xmm8, OWORD PTR [rdi+192]
 9849        jl	L_AES_GCM_init_avx1_calc_iv_2_aesenc_avx_last
 9850        vaesenc	xmm7, xmm7, xmm8
 9851        vaesenc	xmm7, xmm7, [rdi+208]
 9852        vmovdqa	xmm8, OWORD PTR [rdi+224]
 9853L_AES_GCM_init_avx1_calc_iv_2_aesenc_avx_last:
 9854        vaesenclast	xmm7, xmm7, xmm8
 9855        vmovdqu	xmm15, xmm7
 9856L_AES_GCM_init_avx1_iv_done:
 9857        vmovdqa	OWORD PTR [r9], xmm15
 9858        vpshufb	xmm4, xmm4, OWORD PTR L_avx1_aes_gcm_bswap_epi64
 9859        vpaddd	xmm4, xmm4, OWORD PTR L_avx1_aes_gcm_one
 9860        vmovdqa	OWORD PTR [rax], xmm5
 9861        vmovdqa	OWORD PTR [r8], xmm4
 9862        vmovdqu	xmm6, OWORD PTR [rsp+16]
 9863        vmovdqu	xmm7, OWORD PTR [rsp+32]
 9864        vmovdqu	xmm8, OWORD PTR [rsp+48]
 9865        vmovdqu	xmm15, OWORD PTR [rsp+64]
 9866        add	rsp, 80
 9867        pop	r13
 9868        pop	r12
 9869        pop	rsi
 9870        pop	rdi
 9871        ret
 9872AES_GCM_init_avx1 ENDP
 9873_TEXT ENDS
 9874_TEXT SEGMENT READONLY PARA
 9875AES_GCM_aad_update_avx1 PROC
 9876        mov	rax, rcx
 9877        sub	rsp, 32
 9878        vmovdqu	OWORD PTR [rsp], xmm6
 9879        vmovdqu	OWORD PTR [rsp+16], xmm7
 9880        vmovdqa	xmm5, OWORD PTR [r8]
 9881        vmovdqa	xmm6, OWORD PTR [r9]
 9882        xor	ecx, ecx
 9883L_AES_GCM_aad_update_avx1_16_loop:
 9884        vmovdqu	xmm7, OWORD PTR [rax+rcx]
 9885        vpshufb	xmm7, xmm7, OWORD PTR L_avx1_aes_gcm_bswap_mask
 9886        vpxor	xmm5, xmm5, xmm7
 9887        ; ghash_gfmul_avx
 9888        vpshufd	xmm1, xmm5, 78
 9889        vpshufd	xmm2, xmm6, 78
 9890        vpclmulqdq	xmm3, xmm6, xmm5, 17
 9891        vpclmulqdq	xmm0, xmm6, xmm5, 0
 9892        vpxor	xmm1, xmm1, xmm5
 9893        vpxor	xmm2, xmm2, xmm6
 9894        vpclmulqdq	xmm1, xmm1, xmm2, 0
 9895        vpxor	xmm1, xmm1, xmm0
 9896        vpxor	xmm1, xmm1, xmm3
 9897        vmovdqa	xmm4, xmm0
 9898        vmovdqa	xmm5, xmm3
 9899        vpslldq	xmm2, xmm1, 8
 9900        vpsrldq	xmm1, xmm1, 8
 9901        vpxor	xmm4, xmm4, xmm2
 9902        vpxor	xmm5, xmm5, xmm1
 9903        vpsrld	xmm0, xmm4, 31
 9904        vpsrld	xmm1, xmm5, 31
 9905        vpslld	xmm4, xmm4, 1
 9906        vpslld	xmm5, xmm5, 1
 9907        vpsrldq	xmm2, xmm0, 12
 9908        vpslldq	xmm0, xmm0, 4
 9909        vpslldq	xmm1, xmm1, 4
 9910        vpor	xmm5, xmm5, xmm2
 9911        vpor	xmm4, xmm4, xmm0
 9912        vpor	xmm5, xmm5, xmm1
 9913        vpslld	xmm0, xmm4, 31
 9914        vpslld	xmm1, xmm4, 30
 9915        vpslld	xmm2, xmm4, 25
 9916        vpxor	xmm0, xmm0, xmm1
 9917        vpxor	xmm0, xmm0, xmm2
 9918        vmovdqa	xmm1, xmm0
 9919        vpsrldq	xmm1, xmm1, 4
 9920        vpslldq	xmm0, xmm0, 12
 9921        vpxor	xmm4, xmm4, xmm0
 9922        vpsrld	xmm2, xmm4, 1
 9923        vpsrld	xmm3, xmm4, 2
 9924        vpsrld	xmm0, xmm4, 7
 9925        vpxor	xmm2, xmm2, xmm3
 9926        vpxor	xmm2, xmm2, xmm0
 9927        vpxor	xmm2, xmm2, xmm1
 9928        vpxor	xmm2, xmm2, xmm4
 9929        vpxor	xmm5, xmm5, xmm2
 9930        add	ecx, 16
 9931        cmp	ecx, edx
 9932        jl	L_AES_GCM_aad_update_avx1_16_loop
 9933        vmovdqa	OWORD PTR [r8], xmm5
 9934        vmovdqu	xmm6, OWORD PTR [rsp]
 9935        vmovdqu	xmm7, OWORD PTR [rsp+16]
 9936        add	rsp, 32
 9937        ret
 9938AES_GCM_aad_update_avx1 ENDP
 9939_TEXT ENDS
 9940_TEXT SEGMENT READONLY PARA
 9941AES_GCM_encrypt_block_avx1 PROC
 9942        mov	r10, r8
 9943        mov	r11, r9
 9944        mov	rax, QWORD PTR [rsp+40]
 9945        vmovdqu	xmm1, OWORD PTR [rax]
 9946        vpshufb	xmm0, xmm1, OWORD PTR L_avx1_aes_gcm_bswap_epi64
 9947        vpaddd	xmm1, xmm1, OWORD PTR L_avx1_aes_gcm_one
 9948        vmovdqu	OWORD PTR [rax], xmm1
 9949        vpxor	xmm0, xmm0, [rcx]
 9950        vaesenc	xmm0, xmm0, [rcx+16]
 9951        vaesenc	xmm0, xmm0, [rcx+32]
 9952        vaesenc	xmm0, xmm0, [rcx+48]
 9953        vaesenc	xmm0, xmm0, [rcx+64]
 9954        vaesenc	xmm0, xmm0, [rcx+80]
 9955        vaesenc	xmm0, xmm0, [rcx+96]
 9956        vaesenc	xmm0, xmm0, [rcx+112]
 9957        vaesenc	xmm0, xmm0, [rcx+128]
 9958        vaesenc	xmm0, xmm0, [rcx+144]
 9959        cmp	edx, 11
 9960        vmovdqa	xmm1, OWORD PTR [rcx+160]
 9961        jl	L_AES_GCM_encrypt_block_avx1_aesenc_block_last
 9962        vaesenc	xmm0, xmm0, xmm1
 9963        vaesenc	xmm0, xmm0, [rcx+176]
 9964        cmp	edx, 13
 9965        vmovdqa	xmm1, OWORD PTR [rcx+192]
 9966        jl	L_AES_GCM_encrypt_block_avx1_aesenc_block_last
 9967        vaesenc	xmm0, xmm0, xmm1
 9968        vaesenc	xmm0, xmm0, [rcx+208]
 9969        vmovdqa	xmm1, OWORD PTR [rcx+224]
 9970L_AES_GCM_encrypt_block_avx1_aesenc_block_last:
 9971        vaesenclast	xmm0, xmm0, xmm1
 9972        vmovdqu	xmm1, OWORD PTR [r11]
 9973        vpxor	xmm0, xmm0, xmm1
 9974        vmovdqu	OWORD PTR [r10], xmm0
 9975        vpshufb	xmm0, xmm0, OWORD PTR L_avx1_aes_gcm_bswap_mask
 9976        vzeroupper
 9977        ret
 9978AES_GCM_encrypt_block_avx1 ENDP
 9979_TEXT ENDS
 9980_TEXT SEGMENT READONLY PARA
 9981AES_GCM_ghash_block_avx1 PROC
 9982        sub	rsp, 32
 9983        vmovdqu	OWORD PTR [rsp], xmm6
 9984        vmovdqu	OWORD PTR [rsp+16], xmm7
 9985        vmovdqa	xmm4, OWORD PTR [rdx]
 9986        vmovdqa	xmm5, OWORD PTR [r8]
 9987        vmovdqu	xmm7, OWORD PTR [rcx]
 9988        vpshufb	xmm7, xmm7, OWORD PTR L_avx1_aes_gcm_bswap_mask
 9989        vpxor	xmm4, xmm4, xmm7
 9990        ; ghash_gfmul_avx
 9991        vpshufd	xmm1, xmm4, 78
 9992        vpshufd	xmm2, xmm5, 78
 9993        vpclmulqdq	xmm3, xmm5, xmm4, 17
 9994        vpclmulqdq	xmm0, xmm5, xmm4, 0
 9995        vpxor	xmm1, xmm1, xmm4
 9996        vpxor	xmm2, xmm2, xmm5
 9997        vpclmulqdq	xmm1, xmm1, xmm2, 0
 9998        vpxor	xmm1, xmm1, xmm0
 9999        vpxor	xmm1, xmm1, xmm3
10000        vmovdqa	xmm6, xmm0
10001        vmovdqa	xmm4, xmm3
10002        vpslldq	xmm2, xmm1, 8
10003        vpsrldq	xmm1, xmm1, 8
10004        vpxor	xmm6, xmm6, xmm2
10005        vpxor	xmm4, xmm4, xmm1
10006        vpsrld	xmm0, xmm6, 31
10007        vpsrld	xmm1, xmm4, 31
10008        vpslld	xmm6, xmm6, 1
10009        vpslld	xmm4, xmm4, 1
10010        vpsrldq	xmm2, xmm0, 12
10011        vpslldq	xmm0, xmm0, 4
10012        vpslldq	xmm1, xmm1, 4
10013        vpor	xmm4, xmm4, xmm2
10014        vpor	xmm6, xmm6, xmm0
10015        vpor	xmm4, xmm4, xmm1
10016        vpslld	xmm0, xmm6, 31
10017        vpslld	xmm1, xmm6, 30
10018        vpslld	xmm2, xmm6, 25
10019        vpxor	xmm0, xmm0, xmm1
10020        vpxor	xmm0, xmm0, xmm2
10021        vmovdqa	xmm1, xmm0
10022        vpsrldq	xmm1, xmm1, 4
10023        vpslldq	xmm0, xmm0, 12
10024        vpxor	xmm6, xmm6, xmm0
10025        vpsrld	xmm2, xmm6, 1
10026        vpsrld	xmm3, xmm6, 2
10027        vpsrld	xmm0, xmm6, 7
10028        vpxor	xmm2, xmm2, xmm3
10029        vpxor	xmm2, xmm2, xmm0
10030        vpxor	xmm2, xmm2, xmm1
10031        vpxor	xmm2, xmm2, xmm6
10032        vpxor	xmm4, xmm4, xmm2
10033        vmovdqa	OWORD PTR [rdx], xmm4
10034        vzeroupper
10035        vmovdqu	xmm6, OWORD PTR [rsp]
10036        vmovdqu	xmm7, OWORD PTR [rsp+16]
10037        add	rsp, 32
10038        ret
10039AES_GCM_ghash_block_avx1 ENDP
10040_TEXT ENDS
10041_TEXT SEGMENT READONLY PARA
10042AES_GCM_encrypt_update_avx1 PROC
10043        push	r13
10044        push	r12
10045        push	r14
10046        push	r15
10047        push	rdi
10048        mov	rax, rcx
10049        mov	r10, r8
10050        mov	r8d, edx
10051        mov	r11, r9
10052        mov	r9d, DWORD PTR [rsp+80]
10053        mov	r12, QWORD PTR [rsp+88]
10054        mov	r14, QWORD PTR [rsp+96]
10055        mov	r15, QWORD PTR [rsp+104]
10056        sub	rsp, 320
10057        vmovdqu	OWORD PTR [rsp+160], xmm6
10058        vmovdqu	OWORD PTR [rsp+176], xmm7
10059        vmovdqu	OWORD PTR [rsp+192], xmm8
10060        vmovdqu	OWORD PTR [rsp+208], xmm9
10061        vmovdqu	OWORD PTR [rsp+224], xmm10
10062        vmovdqu	OWORD PTR [rsp+240], xmm11
10063        vmovdqu	OWORD PTR [rsp+256], xmm12
10064        vmovdqu	OWORD PTR [rsp+272], xmm13
10065        vmovdqu	OWORD PTR [rsp+288], xmm14
10066        vmovdqu	OWORD PTR [rsp+304], xmm15
10067        vmovdqa	xmm6, OWORD PTR [r12]
10068        vmovdqa	xmm5, OWORD PTR [r14]
10069        vpsrlq	xmm9, xmm5, 63
10070        vpsllq	xmm8, xmm5, 1
10071        vpslldq	xmm9, xmm9, 8
10072        vpor	xmm8, xmm8, xmm9
10073        vpshufd	xmm5, xmm5, 255
10074        vpsrad	xmm5, xmm5, 31
10075        vpand	xmm5, xmm5, OWORD PTR L_avx1_aes_gcm_mod2_128
10076        vpxor	xmm5, xmm5, xmm8
10077        xor	edi, edi
10078        cmp	r9d, 128
10079        mov	r13d, r9d
10080        jl	L_AES_GCM_encrypt_update_avx1_done_128
10081        and	r13d, 4294967168
10082        vmovdqa	xmm2, xmm6
10083        ; H ^ 1
10084        vmovdqu	OWORD PTR [rsp], xmm5
10085        ; H ^ 2
10086        vpclmulqdq	xmm8, xmm5, xmm5, 0
10087        vpclmulqdq	xmm0, xmm5, xmm5, 17
10088        vpslld	xmm12, xmm8, 31
10089        vpslld	xmm13, xmm8, 30
10090        vpslld	xmm14, xmm8, 25
10091        vpxor	xmm12, xmm12, xmm13
10092        vpxor	xmm12, xmm12, xmm14
10093        vpsrldq	xmm13, xmm12, 4
10094        vpslldq	xmm12, xmm12, 12
10095        vpxor	xmm8, xmm8, xmm12
10096        vpsrld	xmm14, xmm8, 1
10097        vpsrld	xmm10, xmm8, 2
10098        vpsrld	xmm9, xmm8, 7
10099        vpxor	xmm14, xmm14, xmm10
10100        vpxor	xmm14, xmm14, xmm9
10101        vpxor	xmm14, xmm14, xmm13
10102        vpxor	xmm14, xmm14, xmm8
10103        vpxor	xmm0, xmm0, xmm14
10104        vmovdqu	OWORD PTR [rsp+16], xmm0
10105        ; H ^ 3
10106        ; ghash_gfmul_red_avx
10107        vpshufd	xmm9, xmm5, 78
10108        vpshufd	xmm10, xmm0, 78
10109        vpclmulqdq	xmm11, xmm0, xmm5, 17
10110        vpclmulqdq	xmm8, xmm0, xmm5, 0
10111        vpxor	xmm9, xmm9, xmm5
10112        vpxor	xmm10, xmm10, xmm0
10113        vpclmulqdq	xmm9, xmm9, xmm10, 0
10114        vpxor	xmm9, xmm9, xmm8
10115        vpxor	xmm9, xmm9, xmm11
10116        vpslldq	xmm10, xmm9, 8
10117        vpsrldq	xmm9, xmm9, 8
10118        vpxor	xmm8, xmm8, xmm10
10119        vpxor	xmm1, xmm11, xmm9
10120        vpslld	xmm12, xmm8, 31
10121        vpslld	xmm13, xmm8, 30
10122        vpslld	xmm14, xmm8, 25
10123        vpxor	xmm12, xmm12, xmm13
10124        vpxor	xmm12, xmm12, xmm14
10125        vpsrldq	xmm13, xmm12, 4
10126        vpslldq	xmm12, xmm12, 12
10127        vpxor	xmm8, xmm8, xmm12
10128        vpsrld	xmm14, xmm8, 1
10129        vpsrld	xmm10, xmm8, 2
10130        vpsrld	xmm9, xmm8, 7
10131        vpxor	xmm14, xmm14, xmm10
10132        vpxor	xmm14, xmm14, xmm9
10133        vpxor	xmm14, xmm14, xmm13
10134        vpxor	xmm14, xmm14, xmm8
10135        vpxor	xmm1, xmm1, xmm14
10136        vmovdqu	OWORD PTR [rsp+32], xmm1
10137        ; H ^ 4
10138        vpclmulqdq	xmm8, xmm0, xmm0, 0
10139        vpclmulqdq	xmm3, xmm0, xmm0, 17
10140        vpslld	xmm12, xmm8, 31
10141        vpslld	xmm13, xmm8, 30
10142        vpslld	xmm14, xmm8, 25
10143        vpxor	xmm12, xmm12, xmm13
10144        vpxor	xmm12, xmm12, xmm14
10145        vpsrldq	xmm13, xmm12, 4
10146        vpslldq	xmm12, xmm12, 12
10147        vpxor	xmm8, xmm8, xmm12
10148        vpsrld	xmm14, xmm8, 1
10149        vpsrld	xmm10, xmm8, 2
10150        vpsrld	xmm9, xmm8, 7
10151        vpxor	xmm14, xmm14, xmm10
10152        vpxor	xmm14, xmm14, xmm9
10153        vpxor	xmm14, xmm14, xmm13
10154        vpxor	xmm14, xmm14, xmm8
10155        vpxor	xmm3, xmm3, xmm14
10156        vmovdqu	OWORD PTR [rsp+48], xmm3
10157        ; H ^ 5
10158        ; ghash_gfmul_red_avx
10159        vpshufd	xmm9, xmm0, 78
10160        vpshufd	xmm10, xmm1, 78
10161        vpclmulqdq	xmm11, xmm1, xmm0, 17
10162        vpclmulqdq	xmm8, xmm1, xmm0, 0
10163        vpxor	xmm9, xmm9, xmm0
10164        vpxor	xmm10, xmm10, xmm1
10165        vpclmulqdq	xmm9, xmm9, xmm10, 0
10166        vpxor	xmm9, xmm9, xmm8
10167        vpxor	xmm9, xmm9, xmm11
10168        vpslldq	xmm10, xmm9, 8
10169        vpsrldq	xmm9, xmm9, 8
10170        vpxor	xmm8, xmm8, xmm10
10171        vpxor	xmm7, xmm11, xmm9
10172        vpslld	xmm12, xmm8, 31
10173        vpslld	xmm13, xmm8, 30
10174        vpslld	xmm14, xmm8, 25
10175        vpxor	xmm12, xmm12, xmm13
10176        vpxor	xmm12, xmm12, xmm14
10177        vpsrldq	xmm13, xmm12, 4
10178        vpslldq	xmm12, xmm12, 12
10179        vpxor	xmm8, xmm8, xmm12
10180        vpsrld	xmm14, xmm8, 1
10181        vpsrld	xmm10, xmm8, 2
10182        vpsrld	xmm9, xmm8, 7
10183        vpxor	xmm14, xmm14, xmm10
10184        vpxor	xmm14, xmm14, xmm9
10185        vpxor	xmm14, xmm14, xmm13
10186        vpxor	xmm14, xmm14, xmm8
10187        vpxor	xmm7, xmm7, xmm14
10188        vmovdqu	OWORD PTR [rsp+64], xmm7
10189        ; H ^ 6
10190        vpclmulqdq	xmm8, xmm1, xmm1, 0
10191        vpclmulqdq	xmm7, xmm1, xmm1, 17
10192        vpslld	xmm12, xmm8, 31
10193        vpslld	xmm13, xmm8, 30
10194        vpslld	xmm14, xmm8, 25
10195        vpxor	xmm12, xmm12, xmm13
10196        vpxor	xmm12, xmm12, xmm14
10197        vpsrldq	xmm13, xmm12, 4
10198        vpslldq	xmm12, xmm12, 12
10199        vpxor	xmm8, xmm8, xmm12
10200        vpsrld	xmm14, xmm8, 1
10201        vpsrld	xmm10, xmm8, 2
10202        vpsrld	xmm9, xmm8, 7
10203        vpxor	xmm14, xmm14, xmm10
10204        vpxor	xmm14, xmm14, xmm9
10205        vpxor	xmm14, xmm14, xmm13
10206        vpxor	xmm14, xmm14, xmm8
10207        vpxor	xmm7, xmm7, xmm14
10208        vmovdqu	OWORD PTR [rsp+80], xmm7
10209        ; H ^ 7
10210        ; ghash_gfmul_red_avx
10211        vpshufd	xmm9, xmm1, 78
10212        vpshufd	xmm10, xmm3, 78
10213        vpclmulqdq	xmm11, xmm3, xmm1, 17
10214        vpclmulqdq	xmm8, xmm3, xmm1, 0
10215        vpxor	xmm9, xmm9, xmm1
10216        vpxor	xmm10, xmm10, xmm3
10217        vpclmulqdq	xmm9, xmm9, xmm10, 0
10218        vpxor	xmm9, xmm9, xmm8
10219        vpxor	xmm9, xmm9, xmm11
10220        vpslldq	xmm10, xmm9, 8
10221        vpsrldq	xmm9, xmm9, 8
10222        vpxor	xmm8, xmm8, xmm10
10223        vpxor	xmm7, xmm11, xmm9
10224        vpslld	xmm12, xmm8, 31
10225        vpslld	xmm13, xmm8, 30
10226        vpslld	xmm14, xmm8, 25
10227        vpxor	xmm12, xmm12, xmm13
10228        vpxor	xmm12, xmm12, xmm14
10229        vpsrldq	xmm13, xmm12, 4
10230        vpslldq	xmm12, xmm12, 12
10231        vpxor	xmm8, xmm8, xmm12
10232        vpsrld	xmm14, xmm8, 1
10233        vpsrld	xmm10, xmm8, 2
10234        vpsrld	xmm9, xmm8, 7
10235        vpxor	xmm14, xmm14, xmm10
10236        vpxor	xmm14, xmm14, xmm9
10237        vpxor	xmm14, xmm14, xmm13
10238        vpxor	xmm14, xmm14, xmm8
10239        vpxor	xmm7, xmm7, xmm14
10240        vmovdqu	OWORD PTR [rsp+96], xmm7
10241        ; H ^ 8
10242        vpclmulqdq	xmm8, xmm3, xmm3, 0
10243        vpclmulqdq	xmm7, xmm3, xmm3, 17
10244        vpslld	xmm12, xmm8, 31
10245        vpslld	xmm13, xmm8, 30
10246        vpslld	xmm14, xmm8, 25
10247        vpxor	xmm12, xmm12, xmm13
10248        vpxor	xmm12, xmm12, xmm14
10249        vpsrldq	xmm13, xmm12, 4
10250        vpslldq	xmm12, xmm12, 12
10251        vpxor	xmm8, xmm8, xmm12
10252        vpsrld	xmm14, xmm8, 1
10253        vpsrld	xmm10, xmm8, 2
10254        vpsrld	xmm9, xmm8, 7
10255        vpxor	xmm14, xmm14, xmm10
10256        vpxor	xmm14, xmm14, xmm9
10257        vpxor	xmm14, xmm14, xmm13
10258        vpxor	xmm14, xmm14, xmm8
10259        vpxor	xmm7, xmm7, xmm14
10260        vmovdqu	OWORD PTR [rsp+112], xmm7
10261        ; First 128 bytes of input
10262        vmovdqu	xmm0, OWORD PTR [r15]
10263        vmovdqa	xmm1, OWORD PTR L_avx1_aes_gcm_bswap_epi64
10264        vpshufb	xmm8, xmm0, xmm1
10265        vpaddd	xmm9, xmm0, OWORD PTR L_avx1_aes_gcm_one
10266        vpshufb	xmm9, xmm9, xmm1
10267        vpaddd	xmm10, xmm0, OWORD PTR L_avx1_aes_gcm_two
10268        vpshufb	xmm10, xmm10, xmm1
10269        vpaddd	xmm11, xmm0, OWORD PTR L_avx1_aes_gcm_three
10270        vpshufb	xmm11, xmm11, xmm1
10271        vpaddd	xmm12, xmm0, OWORD PTR L_avx1_aes_gcm_four
10272        vpshufb	xmm12, xmm12, xmm1
10273        vpaddd	xmm13, xmm0, OWORD PTR L_avx1_aes_gcm_five
10274        vpshufb	xmm13, xmm13, xmm1
10275        vpaddd	xmm14, xmm0, OWORD PTR L_avx1_aes_gcm_six
10276        vpshufb	xmm14, xmm14, xmm1
10277        vpaddd	xmm15, xmm0, OWORD PTR L_avx1_aes_gcm_seven
10278        vpshufb	xmm15, xmm15, xmm1
10279        vpaddd	xmm0, xmm0, OWORD PTR L_avx1_aes_gcm_eight
10280        vmovdqa	xmm7, OWORD PTR [rax]
10281        vmovdqu	OWORD PTR [r15], xmm0
10282        vpxor	xmm8, xmm8, xmm7
10283        vpxor	xmm9, xmm9, xmm7
10284        vpxor	xmm10, xmm10, xmm7
10285        vpxor	xmm11, xmm11, xmm7
10286        vpxor	xmm12, xmm12, xmm7
10287        vpxor	xmm13, xmm13, xmm7
10288        vpxor	xmm14, xmm14, xmm7
10289        vpxor	xmm15, xmm15, xmm7
10290        vmovdqa	xmm7, OWORD PTR [rax+16]
10291        vaesenc	xmm8, xmm8, xmm7
10292        vaesenc	xmm9, xmm9, xmm7
10293        vaesenc	xmm10, xmm10, xmm7
10294        vaesenc	xmm11, xmm11, xmm7
10295        vaesenc	xmm12, xmm12, xmm7
10296        vaesenc	xmm13, xmm13, xmm7
10297        vaesenc	xmm14, xmm14, xmm7
10298        vaesenc	xmm15, xmm15, xmm7
10299        vmovdqa	xmm7, OWORD PTR [rax+32]
10300        vaesenc	xmm8, xmm8, xmm7
10301        vaesenc	xmm9, xmm9, xmm7
10302        vaesenc	xmm10, xmm10, xmm7
10303        vaesenc	xmm11, xmm11, xmm7
10304        vaesenc	xmm12, xmm12, xmm7
10305        vaesenc	xmm13, xmm13, xmm7
10306        vaesenc	xmm14, xmm14, xmm7
10307        vaesenc	xmm15, xmm15, xmm7
10308        vmovdqa	xmm7, OWORD PTR [rax+48]
10309        vaesenc	xmm8, xmm8, xmm7
10310        vaesenc	xmm9, xmm9, xmm7
10311        vaesenc	xmm10, xmm10, xmm7
10312        vaesenc	xmm11, xmm11, xmm7
10313        vaesenc	xmm12, xmm12, xmm7
10314        vaesenc	xmm13, xmm13, xmm7
10315        vaesenc	xmm14, xmm14, xmm7
10316        vaesenc	xmm15, xmm15, xmm7
10317        vmovdqa	xmm7, OWORD PTR [rax+64]
10318        vaesenc	xmm8, xmm8, xmm7
10319        vaesenc	xmm9, xmm9, xmm7
10320        vaesenc	xmm10, xmm10, xmm7
10321        vaesenc	xmm11, xmm11, xmm7
10322        vaesenc	xmm12, xmm12, xmm7
10323        vaesenc	xmm13, xmm13, xmm7
10324        vaesenc	xmm14, xmm14, xmm7
10325        vaesenc	xmm15, xmm15, xmm7
10326        vmovdqa	xmm7, OWORD PTR [rax+80]
10327        vaesenc	xmm8, xmm8, xmm7
10328        vaesenc	xmm9, xmm9, xmm7
10329        vaesenc	xmm10, xmm10, xmm7
10330        vaesenc	xmm11, xmm11, xmm7
10331        vaesenc	xmm12, xmm12, xmm7
10332        vaesenc	xmm13, xmm13, xmm7
10333        vaesenc	xmm14, xmm14, xmm7
10334        vaesenc	xmm15, xmm15, xmm7
10335        vmovdqa	xmm7, OWORD PTR [rax+96]
10336        vaesenc	xmm8, xmm8, xmm7
10337        vaesenc	xmm9, xmm9, xmm7
10338        vaesenc	xmm10, xmm10, xmm7
10339        vaesenc	xmm11, xmm11, xmm7
10340        vaesenc	xmm12, xmm12, xmm7
10341        vaesenc	xmm13, xmm13, xmm7
10342        vaesenc	xmm14, xmm14, xmm7
10343        vaesenc	xmm15, xmm15, xmm7
10344        vmovdqa	xmm7, OWORD PTR [rax+112]
10345        vaesenc	xmm8, xmm8, xmm7
10346        vaesenc	xmm9, xmm9, xmm7
10347        vaesenc	xmm10, xmm10, xmm7
10348        vaesenc	xmm11, xmm11, xmm7
10349        vaesenc	xmm12, xmm12, xmm7
10350        vaesenc	xmm13, xmm13, xmm7
10351        vaesenc	xmm14, xmm14, xmm7
10352        vaesenc	xmm15, xmm15, xmm7
10353        vmovdqa	xmm7, OWORD PTR [rax+128]
10354        vaesenc	xmm8, xmm8, xmm7
10355        vaesenc	xmm9, xmm9, xmm7
10356        vaesenc	xmm10, xmm10, xmm7
10357        vaesenc	xmm11, xmm11, xmm7
10358        vaesenc	xmm12, xmm12, xmm7
10359        vaesenc	xmm13, xmm13, xmm7
10360        vaesenc	xmm14, xmm14, xmm7
10361        vaesenc	xmm15, xmm15, xmm7
10362        vmovdqa	xmm7, OWORD PTR [rax+144]
10363        vaesenc	xmm8, xmm8, xmm7
10364        vaesenc	xmm9, xmm9, xmm7
10365        vaesenc	xmm10, xmm10, xmm7
10366        vaesenc	xmm11, xmm11, xmm7
10367        vaesenc	xmm12, xmm12, xmm7
10368        vaesenc	xmm13, xmm13, xmm7
10369        vaesenc	xmm14, xmm14, xmm7
10370        vaesenc	xmm15, xmm15, xmm7
10371        cmp	r8d, 11
10372        vmovdqa	xmm7, OWORD PTR [rax+160]
10373        jl	L_AES_GCM_encrypt_update_avx1_aesenc_128_enc_done
10374        vaesenc	xmm8, xmm8, xmm7
10375        vaesenc	xmm9, xmm9, xmm7
10376        vaesenc	xmm10, xmm10, xmm7
10377        vaesenc	xmm11, xmm11, xmm7
10378        vaesenc	xmm12, xmm12, xmm7
10379        vaesenc	xmm13, xmm13, xmm7
10380        vaesenc	xmm14, xmm14, xmm7
10381        vaesenc	xmm15, xmm15, xmm7
10382        vmovdqa	xmm7, OWORD PTR [rax+176]
10383        vaesenc	xmm8, xmm8, xmm7
10384        vaesenc	xmm9, xmm9, xmm7
10385        vaesenc	xmm10, xmm10, xmm7
10386        vaesenc	xmm11, xmm11, xmm7
10387        vaesenc	xmm12, xmm12, xmm7
10388        vaesenc	xmm13, xmm13, xmm7
10389        vaesenc	xmm14, xmm14, xmm7
10390        vaesenc	xmm15, xmm15, xmm7
10391        cmp	r8d, 13
10392        vmovdqa	xmm7, OWORD PTR [rax+192]
10393        jl	L_AES_GCM_encrypt_update_avx1_aesenc_128_enc_done
10394        vaesenc	xmm8, xmm8, xmm7
10395        vaesenc	xmm9, xmm9, xmm7
10396        vaesenc	xmm10, xmm10, xmm7
10397        vaesenc	xmm11, xmm11, xmm7
10398        vaesenc	xmm12, xmm12, xmm7
10399        vaesenc	xmm13, xmm13, xmm7
10400        vaesenc	xmm14, xmm14, xmm7
10401        vaesenc	xmm15, xmm15, xmm7
10402        vmovdqa	xmm7, OWORD PTR [rax+208]
10403        vaesenc	xmm8, xmm8, xmm7
10404        vaesenc	xmm9, xmm9, xmm7
10405        vaesenc	xmm10, xmm10, xmm7
10406        vaesenc	xmm11, xmm11, xmm7
10407        vaesenc	xmm12, xmm12, xmm7
10408        vaesenc	xmm13, xmm13, xmm7
10409        vaesenc	xmm14, xmm14, xmm7
10410        vaesenc	xmm15, xmm15, xmm7
10411        vmovdqa	xmm7, OWORD PTR [rax+224]
10412L_AES_GCM_encrypt_update_avx1_aesenc_128_enc_done:
10413        vaesenclast	xmm8, xmm8, xmm7
10414        vaesenclast	xmm9, xmm9, xmm7
10415        vmovdqu	xmm0, OWORD PTR [r11]
10416        vmovdqu	xmm1, OWORD PTR [r11+16]
10417        vpxor	xmm8, xmm8, xmm0
10418        vpxor	xmm9, xmm9, xmm1
10419        vmovdqu	OWORD PTR [r10], xmm8
10420        vmovdqu	OWORD PTR [r10+16], xmm9
10421        vaesenclast	xmm10, xmm10, xmm7
10422        vaesenclast	xmm11, xmm11, xmm7
10423        vmovdqu	xmm0, OWORD PTR [r11+32]
10424        vmovdqu	xmm1, OWORD PTR [r11+48]
10425        vpxor	xmm10, xmm10, xmm0
10426        vpxor	xmm11, xmm11, xmm1
10427        vmovdqu	OWORD PTR [r10+32], xmm10
10428        vmovdqu	OWORD PTR [r10+48], xmm11
10429        vaesenclast	xmm12, xmm12, xmm7
10430        vaesenclast	xmm13, xmm13, xmm7
10431        vmovdqu	xmm0, OWORD PTR [r11+64]
10432        vmovdqu	xmm1, OWORD PTR [r11+80]
10433        vpxor	xmm12, xmm12, xmm0
10434        vpxor	xmm13, xmm13, xmm1
10435        vmovdqu	OWORD PTR [r10+64], xmm12
10436        vmovdqu	OWORD PTR [r10+80], xmm13
10437        vaesenclast	xmm14, xmm14, xmm7
10438        vaesenclast	xmm15, xmm15, xmm7
10439        vmovdqu	xmm0, OWORD PTR [r11+96]
10440        vmovdqu	xmm1, OWORD PTR [r11+112]
10441        vpxor	xmm14, xmm14, xmm0
10442        vpxor	xmm15, xmm15, xmm1
10443        vmovdqu	OWORD PTR [r10+96], xmm14
10444        vmovdqu	OWORD PTR [r10+112], xmm15
10445        cmp	r13d, 128
10446        mov	edi, 128
10447        jle	L_AES_GCM_encrypt_update_avx1_end_128
10448        ; More 128 bytes of input
10449L_AES_GCM_encrypt_update_avx1_ghash_128:
10450        lea	rcx, QWORD PTR [r11+rdi]
10451        lea	rdx, QWORD PTR [r10+rdi]
10452        vmovdqu	xmm0, OWORD PTR [r15]
10453        vmovdqa	xmm1, OWORD PTR L_avx1_aes_gcm_bswap_epi64
10454        vpshufb	xmm8, xmm0, xmm1
10455        vpaddd	xmm9, xmm0, OWORD PTR L_avx1_aes_gcm_one
10456        vpshufb	xmm9, xmm9, xmm1
10457        vpaddd	xmm10, xmm0, OWORD PTR L_avx1_aes_gcm_two
10458        vpshufb	xmm10, xmm10, xmm1
10459        vpaddd	xmm11, xmm0, OWORD PTR L_avx1_aes_gcm_three
10460        vpshufb	xmm11, xmm11, xmm1
10461        vpaddd	xmm12, xmm0, OWORD PTR L_avx1_aes_gcm_four
10462        vpshufb	xmm12, xmm12, xmm1
10463        vpaddd	xmm13, xmm0, OWORD PTR L_avx1_aes_gcm_five
10464        vpshufb	xmm13, xmm13, xmm1
10465        vpaddd	xmm14, xmm0, OWORD PTR L_avx1_aes_gcm_six
10466        vpshufb	xmm14, xmm14, xmm1
10467        vpaddd	xmm15, xmm0, OWORD PTR L_avx1_aes_gcm_seven
10468        vpshufb	xmm15, xmm15, xmm1
10469        vpaddd	xmm0, xmm0, OWORD PTR L_avx1_aes_gcm_eight
10470        vmovdqa	xmm7, OWORD PTR [rax]
10471        vmovdqu	OWORD PTR [r15], xmm0
10472        vpxor	xmm8, xmm8, xmm7
10473        vpxor	xmm9, xmm9, xmm7
10474        vpxor	xmm10, xmm10, xmm7
10475        vpxor	xmm11, xmm11, xmm7
10476        vpxor	xmm12, xmm12, xmm7
10477        vpxor	xmm13, xmm13, xmm7
10478        vpxor	xmm14, xmm14, xmm7
10479        vpxor	xmm15, xmm15, xmm7
10480        vmovdqu	xmm7, OWORD PTR [rsp+112]
10481        vmovdqu	xmm0, OWORD PTR [rdx+-128]
10482        vaesenc	xmm8, xmm8, [rax+16]
10483        vpshufb	xmm0, xmm0, OWORD PTR L_avx1_aes_gcm_bswap_mask
10484        vpxor	xmm0, xmm0, xmm2
10485        vpshufd	xmm1, xmm7, 78
10486        vpshufd	xmm5, xmm0, 78
10487        vpxor	xmm1, xmm1, xmm7
10488        vpxor	xmm5, xmm5, xmm0
10489        vpclmulqdq	xmm3, xmm0, xmm7, 17
10490        vaesenc	xmm9, xmm9, [rax+16]
10491        vaesenc	xmm10, xmm10, [rax+16]
10492        vpclmulqdq	xmm2, xmm0, xmm7, 0
10493        vaesenc	xmm11, xmm11, [rax+16]
10494        vaesenc	xmm12, xmm12, [rax+16]
10495        vpclmulqdq	xmm1, xmm1, xmm5, 0
10496        vaesenc	xmm13, xmm13, [rax+16]
10497        vaesenc	xmm14, xmm14, [rax+16]
10498        vaesenc	xmm15, xmm15, [rax+16]
10499        vpxor	xmm1, xmm1, xmm2
10500        vpxor	xmm1, xmm1, xmm3
10501        vmovdqu	xmm7, OWORD PTR [rsp+96]
10502        vmovdqu	xmm0, OWORD PTR [rdx+-112]
10503        vpshufd	xmm4, xmm7, 78
10504        vpshufb	xmm0, xmm0, OWORD PTR L_avx1_aes_gcm_bswap_mask
10505        vaesenc	xmm8, xmm8, [rax+32]
10506        vpxor	xmm4, xmm4, xmm7
10507        vpshufd	xmm5, xmm0, 78
10508        vpxor	xmm5, xmm5, xmm0
10509        vpclmulqdq	xmm6, xmm0, xmm7, 17
10510        vaesenc	xmm9, xmm9, [rax+32]
10511        vaesenc	xmm10, xmm10, [rax+32]
10512        vpclmulqdq	xmm7, xmm0, xmm7, 0
10513        vaesenc	xmm11, xmm11, [rax+32]
10514        vaesenc	xmm12, xmm12, [rax+32]
10515        vpclmulqdq	xmm4, xmm4, xmm5, 0
10516        vaesenc	xmm13, xmm13, [rax+32]
10517        vaesenc	xmm14, xmm14, [rax+32]
10518        vaesenc	xmm15, xmm15, [rax+32]
10519        vpxor	xmm1, xmm1, xmm7
10520        vpxor	xmm2, xmm2, xmm7
10521        vpxor	xmm1, xmm1, xmm6
10522        vpxor	xmm3, xmm3, xmm6
10523        vpxor	xmm1, xmm1, xmm4
10524        vmovdqu	xmm7, OWORD PTR [rsp+80]
10525        vmovdqu	xmm0, OWORD PTR [rdx+-96]
10526        vpshufd	xmm4, xmm7, 78
10527        vpshufb	xmm0, xmm0, OWORD PTR L_avx1_aes_gcm_bswap_mask
10528        vaesenc	xmm8, xmm8, [rax+48]
10529        vpxor	xmm4, xmm4, xmm7
10530        vpshufd	xmm5, xmm0, 78
10531        vpxor	xmm5, xmm5, xmm0
10532        vpclmulqdq	xmm6, xmm0, xmm7, 17
10533        vaesenc	xmm9, xmm9, [rax+48]
10534        vaesenc	xmm10, xmm10, [rax+48]
10535        vpclmulqdq	xmm7, xmm0, xmm7, 0
10536        vaesenc	xmm11, xmm11, [rax+48]
10537        vaesenc	xmm12, xmm12, [rax+48]
10538        vpclmulqdq	xmm4, xmm4, xmm5, 0
10539        vaesenc	xmm13, xmm13, [rax+48]
10540        vaesenc	xmm14, xmm14, [rax+48]
10541        vaesenc	xmm15, xmm15, [rax+48]
10542        vpxor	xmm1, xmm1, xmm7
10543        vpxor	xmm2, xmm2, xmm7
10544        vpxor	xmm1, xmm1, xmm6
10545        vpxor	xmm3, xmm3, xmm6
10546        vpxor	xmm1, xmm1, xmm4
10547        vmovdqu	xmm7, OWORD PTR [rsp+64]
10548        vmovdqu	xmm0, OWORD PTR [rdx+-80]
10549        vpshufd	xmm4, xmm7, 78
10550        vpshufb	xmm0, xmm0, OWORD PTR L_avx1_aes_gcm_bswap_mask
10551        vaesenc	xmm8, xmm8, [rax+64]
10552        vpxor	xmm4, xmm4, xmm7
10553        vpshufd	xmm5, xmm0, 78
10554        vpxor	xmm5, xmm5, xmm0
10555        vpclmulqdq	xmm6, xmm0, xmm7, 17
10556        vaesenc	xmm9, xmm9, [rax+64]
10557        vaesenc	xmm10, xmm10, [rax+64]
10558        vpclmulqdq	xmm7, xmm0, xmm7, 0
10559        vaesenc	xmm11, xmm11, [rax+64]
10560        vaesenc	xmm12, xmm12, [rax+64]
10561        vpclmulqdq	xmm4, xmm4, xmm5, 0
10562        vaesenc	xmm13, xmm13, [rax+64]
10563        vaesenc	xmm14, xmm14, [rax+64]
10564        vaesenc	xmm15, xmm15, [rax+64]
10565        vpxor	xmm1, xmm1, xmm7
10566        vpxor	xmm2, xmm2, xmm7
10567        vpxor	xmm1, xmm1, xmm6
10568        vpxor	xmm3, xmm3, xmm6
10569        vpxor	xmm1, xmm1, xmm4
10570        vmovdqu	xmm7, OWORD PTR [rsp+48]
10571        vmovdqu	xmm0, OWORD PTR [rdx+-64]
10572        vpshufd	xmm4, xmm7, 78
10573        vpshufb	xmm0, xmm0, OWORD PTR L_avx1_aes_gcm_bswap_mask
10574        vaesenc	xmm8, xmm8, [rax+80]
10575        vpxor	xmm4, xmm4, xmm7
10576        vpshufd	xmm5, xmm0, 78
10577        vpxor	xmm5, xmm5, xmm0
10578        vpclmulqdq	xmm6, xmm0, xmm7, 17
10579        vaesenc	xmm9, xmm9, [rax+80]
10580        vaesenc	xmm10, xmm10, [rax+80]
10581        vpclmulqdq	xmm7, xmm0, xmm7, 0
10582        vaesenc	xmm11, xmm11, [rax+80]
10583        vaesenc	xmm12, xmm12, [rax+80]
10584        vpclmulqdq	xmm4, xmm4, xmm5, 0
10585        vaesenc	xmm13, xmm13, [rax+80]
10586        vaesenc	xmm14, xmm14, [rax+80]
10587        vaesenc	xmm15, xmm15, [rax+80]
10588        vpxor	xmm1, xmm1, xmm7
10589        vpxor	xmm2, xmm2, xmm7
10590        vpxor	xmm1, xmm1, xmm6
10591        vpxor	xmm3, xmm3, xmm6
10592        vpxor	xmm1, xmm1, xmm4
10593        vmovdqu	xmm7, OWORD PTR [rsp+32]
10594        vmovdqu	xmm0, OWORD PTR [rdx+-48]
10595        vpshufd	xmm4, xmm7, 78
10596        vpshufb	xmm0, xmm0, OWORD PTR L_avx1_aes_gcm_bswap_mask
10597        vaesenc	xmm8, xmm8, [rax+96]
10598        vpxor	xmm4, xmm4, xmm7
10599        vpshufd	xmm5, xmm0, 78
10600        vpxor	xmm5, xmm5, xmm0
10601        vpclmulqdq	xmm6, xmm0, xmm7, 17
10602        vaesenc	xmm9, xmm9, [rax+96]
10603        vaesenc	xmm10, xmm10, [rax+96]
10604        vpclmulqdq	xmm7, xmm0, xmm7, 0
10605        vaesenc	xmm11, xmm11, [rax+96]
10606        vaesenc	xmm12, xmm12, [rax+96]
10607        vpclmulqdq	xmm4, xmm4, xmm5, 0
10608        vaesenc	xmm13, xmm13, [rax+96]
10609        vaesenc	xmm14, xmm14, [rax+96]
10610        vaesenc	xmm15, xmm15, [rax+96]
10611        vpxor	xmm1, xmm1, xmm7
10612        vpxor	xmm2, xmm2, xmm7
10613        vpxor	xmm1, xmm1, xmm6
10614        vpxor	xmm3, xmm3, xmm6
10615        vpxor	xmm1, xmm1, xmm4
10616        vmovdqu	xmm7, OWORD PTR [rsp+16]
10617        vmovdqu	xmm0, OWORD PTR [rdx+-32]
10618        vpshufd	xmm4, xmm7, 78
10619        vpshufb	xmm0, xmm0, OWORD PTR L_avx1_aes_gcm_bswap_mask
10620        vaesenc	xmm8, xmm8, [rax+112]
10621        vpxor	xmm4, xmm4, xmm7
10622        vpshufd	xmm5, xmm0, 78
10623        vpxor	xmm5, xmm5, xmm0
10624        vpclmulqdq	xmm6, xmm0, xmm7, 17
10625        vaesenc	xmm9, xmm9, [rax+112]
10626        vaesenc	xmm10, xmm10, [rax+112]
10627        vpclmulqdq	xmm7, xmm0, xmm7, 0
10628        vaesenc	xmm11, xmm11, [rax+112]
10629        vaesenc	xmm12, xmm12, [rax+112]
10630        vpclmulqdq	xmm4, xmm4, xmm5, 0
10631        vaesenc	xmm13, xmm13, [rax+112]
10632        vaesenc	xmm14, xmm14, [rax+112]
10633        vaesenc	xmm15, xmm15, [rax+112]
10634        vpxor	xmm1, xmm1, xmm7
10635        vpxor	xmm2, xmm2, xmm7
10636        vpxor	xmm1, xmm1, xmm6
10637        vpxor	xmm3, xmm3, xmm6
10638        vpxor	xmm1, xmm1, xmm4
10639        vmovdqu	xmm7, OWORD PTR [rsp]
10640        vmovdqu	xmm0, OWORD PTR [rdx+-16]
10641        vpshufd	xmm4, xmm7, 78
10642        vpshufb	xmm0, xmm0, OWORD PTR L_avx1_aes_gcm_bswap_mask
10643        vaesenc	xmm8, xmm8, [rax+128]
10644        vpxor	xmm4, xmm4, xmm7
10645        vpshufd	xmm5, xmm0, 78
10646        vpxor	xmm5, xmm5, xmm0
10647        vpclmulqdq	xmm6, xmm0, xmm7, 17
10648        vaesenc	xmm9, xmm9, [rax+128]
10649        vaesenc	xmm10, xmm10, [rax+128]
10650        vpclmulqdq	xmm7, xmm0, xmm7, 0
10651        vaesenc	xmm11, xmm11, [rax+128]
10652        vaesenc	xmm12, xmm12, [rax+128]
10653        vpclmulqdq	xmm4, xmm4, xmm5, 0
10654        vaesenc	xmm13, xmm13, [rax+128]
10655        vaesenc	xmm14, xmm14, [rax+128]
10656        vaesenc	xmm15, xmm15, [rax+128]
10657        vpxor	xmm1, xmm1, xmm7
10658        vpxor	xmm2, xmm2, xmm7
10659        vpxor	xmm1, xmm1, xmm6
10660        vpxor	xmm3, xmm3, xmm6
10661        vpxor	xmm1, xmm1, xmm4
10662        vpslldq	xmm5, xmm1, 8
10663        vpsrldq	xmm1, xmm1, 8
10664        vaesenc	xmm8, xmm8, [rax+144]
10665        vpxor	xmm2, xmm2, xmm5
10666        vpxor	xmm3, xmm3, xmm1
10667        vaesenc	xmm9, xmm9, [rax+144]
10668        vpslld	xmm7, xmm2, 31
10669        vpslld	xmm4, xmm2, 30
10670        vpslld	xmm5, xmm2, 25
10671        vaesenc	xmm10, xmm10, [rax+144]
10672        vpxor	xmm7, xmm7, xmm4
10673        vpxor	xmm7, xmm7, xmm5
10674        vaesenc	xmm11, xmm11, [rax+144]
10675        vpsrldq	xmm4, xmm7, 4
10676        vpslldq	xmm7, xmm7, 12
10677        vaesenc	xmm12, xmm12, [rax+144]
10678        vpxor	xmm2, xmm2, xmm7
10679        vpsrld	xmm5, xmm2, 1
10680        vaesenc	xmm13, xmm13, [rax+144]
10681        vpsrld	xmm1, xmm2, 2
10682        vpsrld	xmm0, xmm2, 7
10683        vaesenc	xmm14, xmm14, [rax+144]
10684        vpxor	xmm5, xmm5, xmm1
10685        vpxor	xmm5, xmm5, xmm0
10686        vaesenc	xmm15, xmm15, [rax+144]
10687        vpxor	xmm5, xmm5, xmm4
10688        vpxor	xmm2, xmm2, xmm5
10689        vpxor	xmm2, xmm2, xmm3
10690        cmp	r8d, 11
10691        vmovdqa	xmm7, OWORD PTR [rax+160]
10692        jl	L_AES_GCM_encrypt_update_avx1_aesenc_128_ghash_avx_done
10693        vaesenc	xmm8, xmm8, xmm7
10694        vaesenc	xmm9, xmm9, xmm7
10695        vaesenc	xmm10, xmm10, xmm7
10696        vaesenc	xmm11, xmm11, xmm7
10697        vaesenc	xmm12, xmm12, xmm7
10698        vaesenc	xmm13, xmm13, xmm7
10699        vaesenc	xmm14, xmm14, xmm7
10700        vaesenc	xmm15, xmm15, xmm7
10701        vmovdqa	xmm7, OWORD PTR [rax+176]
10702        vaesenc	xmm8, xmm8, xmm7
10703        vaesenc	xmm9, xmm9, xmm7
10704        vaesenc	xmm10, xmm10, xmm7
10705        vaesenc	xmm11, xmm11, xmm7
10706        vaesenc	xmm12, xmm12, xmm7
10707        vaesenc	xmm13, xmm13, xmm7
10708        vaesenc	xmm14, xmm14, xmm7
10709        vaesenc	xmm15, xmm15, xmm7
10710        cmp	r8d, 13
10711        vmovdqa	xmm7, OWORD PTR [rax+192]
10712        jl	L_AES_GCM_encrypt_update_avx1_aesenc_128_ghash_avx_done
10713        vaesenc	xmm8, xmm8, xmm7
10714        vaesenc	xmm9, xmm9, xmm7
10715        vaesenc	xmm10, xmm10, xmm7
10716        vaesenc	xmm11, xmm11, xmm7
10717        vaesenc	xmm12, xmm12, xmm7
10718        vaesenc	xmm13, xmm13, xmm7
10719        vaesenc	xmm14, xmm14, xmm7
10720        vaesenc	xmm15, xmm15, xmm7
10721        vmovdqa	xmm7, OWORD PTR [rax+208]
10722        vaesenc	xmm8, xmm8, xmm7
10723        vaesenc	xmm9, xmm9, xmm7
10724        vaesenc	xmm10, xmm10, xmm7
10725        vaesenc	xmm11, xmm11, xmm7
10726        vaesenc	xmm12, xmm12, xmm7
10727        vaesenc	xmm13, xmm13, xmm7
10728        vaesenc	xmm14, xmm14, xmm7
10729        vaesenc	xmm15, xmm15, xmm7
10730        vmovdqa	xmm7, OWORD PTR [rax+224]
10731L_AES_GCM_encrypt_update_avx1_aesenc_128_ghash_avx_done:
10732        vaesenclast	xmm8, xmm8, xmm7
10733        vaesenclast	xmm9, xmm9, xmm7
10734        vmovdqu	xmm0, OWORD PTR [rcx]
10735        vmovdqu	xmm1, OWORD PTR [rcx+16]
10736        vpxor	xmm8, xmm8, xmm0
10737        vpxor	xmm9, xmm9, xmm1
10738        vmovdqu	OWORD PTR [rdx], xmm8
10739        vmovdqu	OWORD PTR [rdx+16], xmm9
10740        vaesenclast	xmm10, xmm10, xmm7
10741        vaesenclast	xmm11, xmm11, xmm7
10742        vmovdqu	xmm0, OWORD PTR [rcx+32]
10743        vmovdqu	xmm1, OWORD PTR [rcx+48]
10744        vpxor	xmm10, xmm10, xmm0
10745        vpxor	xmm11, xmm11, xmm1
10746        vmovdqu	OWORD PTR [rdx+32], xmm10
10747        vmovdqu	OWORD PTR [rdx+48], xmm11
10748        vaesenclast	xmm12, xmm12, xmm7
10749        vaesenclast	xmm13, xmm13, xmm7
10750        vmovdqu	xmm0, OWORD PTR [rcx+64]
10751        vmovdqu	xmm1, OWORD PTR [rcx+80]
10752        vpxor	xmm12, xmm12, xmm0
10753        vpxor	xmm13, xmm13, xmm1
10754        vmovdqu	OWORD PTR [rdx+64], xmm12
10755        vmovdqu	OWORD PTR [rdx+80], xmm13
10756        vaesenclast	xmm14, xmm14, xmm7
10757        vaesenclast	xmm15, xmm15, xmm7
10758        vmovdqu	xmm0, OWORD PTR [rcx+96]
10759        vmovdqu	xmm1, OWORD PTR [rcx+112]
10760        vpxor	xmm14, xmm14, xmm0
10761        vpxor	xmm15, xmm15, xmm1
10762        vmovdqu	OWORD PTR [rdx+96], xmm14
10763        vmovdqu	OWORD PTR [rdx+112], xmm15
10764        add	edi, 128
10765        cmp	edi, r13d
10766        jl	L_AES_GCM_encrypt_update_avx1_ghash_128
10767L_AES_GCM_encrypt_update_avx1_end_128:
10768        vmovdqa	xmm4, OWORD PTR L_avx1_aes_gcm_bswap_mask
10769        vpshufb	xmm8, xmm8, xmm4
10770        vpshufb	xmm9, xmm9, xmm4
10771        vpshufb	xmm10, xmm10, xmm4
10772        vpshufb	xmm11, xmm11, xmm4
10773        vpxor	xmm8, xmm8, xmm2
10774        vpshufb	xmm12, xmm12, xmm4
10775        vpshufb	xmm13, xmm13, xmm4
10776        vpshufb	xmm14, xmm14, xmm4
10777        vpshufb	xmm15, xmm15, xmm4
10778        vmovdqu	xmm7, OWORD PTR [rsp]
10779        vmovdqu	xmm5, OWORD PTR [rsp+16]
10780        ; ghash_gfmul_avx
10781        vpshufd	xmm1, xmm15, 78
10782        vpshufd	xmm2, xmm7, 78
10783        vpclmulqdq	xmm3, xmm7, xmm15, 17
10784        vpclmulqdq	xmm0, xmm7, xmm15, 0
10785        vpxor	xmm1, xmm1, xmm15
10786        vpxor	xmm2, xmm2, xmm7
10787        vpclmulqdq	xmm1, xmm1, xmm2, 0
10788        vpxor	xmm1, xmm1, xmm0
10789        vpxor	xmm1, xmm1, xmm3
10790        vmovdqa	xmm4, xmm0
10791        vmovdqa	xmm6, xmm3
10792        vpslldq	xmm2, xmm1, 8
10793        vpsrldq	xmm1, xmm1, 8
10794        vpxor	xmm4, xmm4, xmm2
10795        vpxor	xmm6, xmm6, xmm1
10796        ; ghash_gfmul_xor_avx
10797        vpshufd	xmm1, xmm14, 78
10798        vpshufd	xmm2, xmm5, 78
10799        vpclmulqdq	xmm3, xmm5, xmm14, 17
10800        vpclmulqdq	xmm0, xmm5, xmm14, 0
10801        vpxor	xmm1, xmm1, xmm14
10802        vpxor	xmm2, xmm2, xmm5
10803        vpclmulqdq	xmm1, xmm1, xmm2, 0
10804        vpxor	xmm1, xmm1, xmm0
10805        vpxor	xmm1, xmm1, xmm3
10806        vpxor	xmm4, xmm4, xmm0
10807        vpxor	xmm6, xmm6, xmm3
10808        vpslldq	xmm2, xmm1, 8
10809        vpsrldq	xmm1, xmm1, 8
10810        vpxor	xmm4, xmm4, xmm2
10811        vpxor	xmm6, xmm6, xmm1
10812        vmovdqu	xmm7, OWORD PTR [rsp+32]
10813        vmovdqu	xmm5, OWORD PTR [rsp+48]
10814        ; ghash_gfmul_xor_avx
10815        vpshufd	xmm1, xmm13, 78
10816        vpshufd	xmm2, xmm7, 78
10817        vpclmulqdq	xmm3, xmm7, xmm13, 17
10818        vpclmulqdq	xmm0, xmm7, xmm13, 0
10819        vpxor	xmm1, xmm1, xmm13
10820        vpxor	xmm2, xmm2, xmm7
10821        vpclmulqdq	xmm1, xmm1, xmm2, 0
10822        vpxor	xmm1, xmm1, xmm0
10823        vpxor	xmm1, xmm1, xmm3
10824        vpxor	xmm4, xmm4, xmm0
10825        vpxor	xmm6, xmm6, xmm3
10826        vpslldq	xmm2, xmm1, 8
10827        vpsrldq	xmm1, xmm1, 8
10828        vpxor	xmm4, xmm4, xmm2
10829        vpxor	xmm6, xmm6, xmm1
10830        ; ghash_gfmul_xor_avx
10831        vpshufd	xmm1, xmm12, 78
10832        vpshufd	xmm2, xmm5, 78
10833        vpclmulqdq	xmm3, xmm5, xmm12, 17
10834        vpclmulqdq	xmm0, xmm5, xmm12, 0
10835        vpxor	xmm1, xmm1, xmm12
10836        vpxor	xmm2, xmm2, xmm5
10837        vpclmulqdq	xmm1, xmm1, xmm2, 0
10838        vpxor	xmm1, xmm1, xmm0
10839        vpxor	xmm1, xmm1, xmm3
10840        vpxor	xmm4, xmm4, xmm0
10841        vpxor	xmm6, xmm6, xmm3
10842        vpslldq	xmm2, xmm1, 8
10843        vpsrldq	xmm1, xmm1, 8
10844        vpxor	xmm4, xmm4, xmm2
10845        vpxor	xmm6, xmm6, xmm1
10846        vmovdqu	xmm7, OWORD PTR [rsp+64]
10847        vmovdqu	xmm5, OWORD PTR [rsp+80]
10848        ; ghash_gfmul_xor_avx
10849        vpshufd	xmm1, xmm11, 78
10850        vpshufd	xmm2, xmm7, 78
10851        vpclmulqdq	xmm3, xmm7, xmm11, 17
10852        vpclmulqdq	xmm0, xmm7, xmm11, 0
10853        vpxor	xmm1, xmm1, xmm11
10854        vpxor	xmm2, xmm2, xmm7
10855        vpclmulqdq	xmm1, xmm1, xmm2, 0
10856        vpxor	xmm1, xmm1, xmm0
10857        vpxor	xmm1, xmm1, xmm3
10858        vpxor	xmm4, xmm4, xmm0
10859        vpxor	xmm6, xmm6, xmm3
10860        vpslldq	xmm2, xmm1, 8
10861        vpsrldq	xmm1, xmm1, 8
10862        vpxor	xmm4, xmm4, xmm2
10863        vpxor	xmm6, xmm6, xmm1
10864        ; ghash_gfmul_xor_avx
10865        vpshufd	xmm1, xmm10, 78
10866        vpshufd	xmm2, xmm5, 78
10867        vpclmulqdq	xmm3, xmm5, xmm10, 17
10868        vpclmulqdq	xmm0, xmm5, xmm10, 0
10869        vpxor	xmm1, xmm1, xmm10
10870        vpxor	xmm2, xmm2, xmm5
10871        vpclmulqdq	xmm1, xmm1, xmm2, 0
10872        vpxor	xmm1, xmm1, xmm0
10873        vpxor	xmm1, xmm1, xmm3
10874        vpxor	xmm4, xmm4, xmm0
10875        vpxor	xmm6, xmm6, xmm3
10876        vpslldq	xmm2, xmm1, 8
10877        vpsrldq	xmm1, xmm1, 8
10878        vpxor	xmm4, xmm4, xmm2
10879        vpxor	xmm6, xmm6, xmm1
10880        vmovdqu	xmm7, OWORD PTR [rsp+96]
10881        vmovdqu	xmm5, OWORD PTR [rsp+112]
10882        ; ghash_gfmul_xor_avx
10883        vpshufd	xmm1, xmm9, 78
10884        vpshufd	xmm2, xmm7, 78
10885        vpclmulqdq	xmm3, xmm7, xmm9, 17
10886        vpclmulqdq	xmm0, xmm7, xmm9, 0
10887        vpxor	xmm1, xmm1, xmm9
10888        vpxor	xmm2, xmm2, xmm7
10889        vpclmulqdq	xmm1, xmm1, xmm2, 0
10890        vpxor	xmm1, xmm1, xmm0
10891        vpxor	xmm1, xmm1, xmm3
10892        vpxor	xmm4, xmm4, xmm0
10893        vpxor	xmm6, xmm6, xmm3
10894        vpslldq	xmm2, xmm1, 8
10895        vpsrldq	xmm1, xmm1, 8
10896        vpxor	xmm4, xmm4, xmm2
10897        vpxor	xmm6, xmm6, xmm1
10898        ; ghash_gfmul_xor_avx
10899        vpshufd	xmm1, xmm8, 78
10900        vpshufd	xmm2, xmm5, 78
10901        vpclmulqdq	xmm3, xmm5, xmm8, 17
10902        vpclmulqdq	xmm0, xmm5, xmm8, 0
10903        vpxor	xmm1, xmm1, xmm8
10904        vpxor	xmm2, xmm2, xmm5
10905        vpclmulqdq	xmm1, xmm1, xmm2, 0
10906        vpxor	xmm1, xmm1, xmm0
10907        vpxor	xmm1, xmm1, xmm3
10908        vpxor	xmm4, xmm4, xmm0
10909        vpxor	xmm6, xmm6, xmm3
10910        vpslldq	xmm2, xmm1, 8
10911        vpsrldq	xmm1, xmm1, 8
10912        vpxor	xmm4, xmm4, xmm2
10913        vpxor	xmm6, xmm6, xmm1
10914        vpslld	xmm0, xmm4, 31
10915        vpslld	xmm1, xmm4, 30
10916        vpslld	xmm2, xmm4, 25
10917        vpxor	xmm0, xmm0, xmm1
10918        vpxor	xmm0, xmm0, xmm2
10919        vmovdqa	xmm1, xmm0
10920        vpsrldq	xmm1, xmm1, 4
10921        vpslldq	xmm0, xmm0, 12
10922        vpxor	xmm4, xmm4, xmm0
10923        vpsrld	xmm2, xmm4, 1
10924        vpsrld	xmm3, xmm4, 2
10925        vpsrld	xmm0, xmm4, 7
10926        vpxor	xmm2, xmm2, xmm3
10927        vpxor	xmm2, xmm2, xmm0
10928        vpxor	xmm2, xmm2, xmm1
10929        vpxor	xmm2, xmm2, xmm4
10930        vpxor	xmm6, xmm6, xmm2
10931        vmovdqu	xmm5, OWORD PTR [rsp]
10932L_AES_GCM_encrypt_update_avx1_done_128:
10933        mov	edx, r9d
10934        cmp	edi, edx
10935        jge	L_AES_GCM_encrypt_update_avx1_done_enc
10936        mov	r13d, r9d
10937        and	r13d, 4294967280
10938        cmp	edi, r13d
10939        jge	L_AES_GCM_encrypt_update_avx1_last_block_done
10940        vmovdqu	xmm9, OWORD PTR [r15]
10941        vpshufb	xmm8, xmm9, OWORD PTR L_avx1_aes_gcm_bswap_epi64
10942        vpaddd	xmm9, xmm9, OWORD PTR L_avx1_aes_gcm_one
10943        vmovdqu	OWORD PTR [r15], xmm9
10944        vpxor	xmm8, xmm8, [rax]
10945        vaesenc	xmm8, xmm8, [rax+16]
10946        vaesenc	xmm8, xmm8, [rax+32]
10947        vaesenc	xmm8, xmm8, [rax+48]
10948        vaesenc	xmm8, xmm8, [rax+64]
10949        vaesenc	xmm8, xmm8, [rax+80]
10950        vaesenc	xmm8, xmm8, [rax+96]
10951        vaesenc	xmm8, xmm8, [rax+112]
10952        vaesenc	xmm8, xmm8, [rax+128]
10953        vaesenc	xmm8, xmm8, [rax+144]
10954        cmp	r8d, 11
10955        vmovdqa	xmm9, OWORD PTR [rax+160]
10956        jl	L_AES_GCM_encrypt_update_avx1_aesenc_block_last
10957        vaesenc	xmm8, xmm8, xmm9
10958        vaesenc	xmm8, xmm8, [rax+176]
10959        cmp	r8d, 13
10960        vmovdqa	xmm9, OWORD PTR [rax+192]
10961        jl	L_AES_GCM_encrypt_update_avx1_aesenc_block_last
10962        vaesenc	xmm8, xmm8, xmm9
10963        vaesenc	xmm8, xmm8, [rax+208]
10964        vmovdqa	xmm9, OWORD PTR [rax+224]
10965L_AES_GCM_encrypt_update_avx1_aesenc_block_last:
10966        vaesenclast	xmm8, xmm8, xmm9
10967        vmovdqu	xmm9, OWORD PTR [r11+rdi]
10968        vpxor	xmm8, xmm8, xmm9
10969        vmovdqu	OWORD PTR [r10+rdi], xmm8
10970        vpshufb	xmm8, xmm8, OWORD PTR L_avx1_aes_gcm_bswap_mask
10971        vpxor	xmm6, xmm6, xmm8
10972        add	edi, 16
10973        cmp	edi, r13d
10974        jge	L_AES_GCM_encrypt_update_avx1_last_block_ghash
10975L_AES_GCM_encrypt_update_avx1_last_block_start:
10976        vmovdqu	xmm13, OWORD PTR [r11+rdi]
10977        vmovdqu	xmm9, OWORD PTR [r15]
10978        vpshufb	xmm8, xmm9, OWORD PTR L_avx1_aes_gcm_bswap_epi64
10979        vpaddd	xmm9, xmm9, OWORD PTR L_avx1_aes_gcm_one
10980        vmovdqu	OWORD PTR [r15], xmm9
10981        vpxor	xmm8, xmm8, [rax]
10982        vpclmulqdq	xmm10, xmm6, xmm5, 16
10983        vaesenc	xmm8, xmm8, [rax+16]
10984        vaesenc	xmm8, xmm8, [rax+32]
10985        vpclmulqdq	xmm11, xmm6, xmm5, 1
10986        vaesenc	xmm8, xmm8, [rax+48]
10987        vaesenc	xmm8, xmm8, [rax+64]
10988        vpclmulqdq	xmm12, xmm6, xmm5, 0
10989        vaesenc	xmm8, xmm8, [rax+80]
10990        vpclmulqdq	xmm1, xmm6, xmm5, 17
10991        vaesenc	xmm8, xmm8, [rax+96]
10992        vpxor	xmm10, xmm10, xmm11
10993        vpslldq	xmm2, xmm10, 8
10994        vpsrldq	xmm10, xmm10, 8
10995        vaesenc	xmm8, xmm8, [rax+112]
10996        vpxor	xmm2, xmm2, xmm12
10997        vpxor	xmm3, xmm1, xmm10
10998        vmovdqa	xmm0, OWORD PTR L_avx1_aes_gcm_mod2_128
10999        vpclmulqdq	xmm11, xmm2, xmm0, 16
11000        vaesenc	xmm8, xmm8, [rax+128]
11001        vpshufd	xmm10, xmm2, 78
11002        vpxor	xmm10, xmm10, xmm11
11003        vpclmulqdq	xmm11, xmm10, xmm0, 16
11004        vaesenc	xmm8, xmm8, [rax+144]
11005        vpshufd	xmm10, xmm10, 78
11006        vpxor	xmm10, xmm10, xmm11
11007        vpxor	xmm6, xmm10, xmm3
11008        cmp	r8d, 11
11009        vmovdqa	xmm9, OWORD PTR [rax+160]
11010        jl	L_AES_GCM_encrypt_update_avx1_aesenc_gfmul_last
11011        vaesenc	xmm8, xmm8, xmm9
11012        vaesenc	xmm8, xmm8, [rax+176]
11013        cmp	r8d, 13
11014        vmovdqa	xmm9, OWORD PTR [rax+192]
11015        jl	L_AES_GCM_encrypt_update_avx1_aesenc_gfmul_last
11016        vaesenc	xmm8, xmm8, xmm9
11017        vaesenc	xmm8, xmm8, [rax+208]
11018        vmovdqa	xmm9, OWORD PTR [rax+224]
11019L_AES_GCM_encrypt_update_avx1_aesenc_gfmul_last:
11020        vaesenclast	xmm8, xmm8, xmm9
11021        vmovdqa	xmm0, xmm13
11022        vpxor	xmm8, xmm8, xmm0
11023        vmovdqu	OWORD PTR [r10+rdi], xmm8
11024        vpshufb	xmm8, xmm8, OWORD PTR L_avx1_aes_gcm_bswap_mask
11025        add	edi, 16
11026        vpxor	xmm6, xmm6, xmm8
11027        cmp	edi, r13d
11028        jl	L_AES_GCM_encrypt_update_avx1_last_block_start
11029L_AES_GCM_encrypt_update_avx1_last_block_ghash:
11030        ; ghash_gfmul_red_avx
11031        vpshufd	xmm9, xmm5, 78
11032        vpshufd	xmm10, xmm6, 78
11033        vpclmulqdq	xmm11, xmm6, xmm5, 17
11034        vpclmulqdq	xmm8, xmm6, xmm5, 0
11035        vpxor	xmm9, xmm9, xmm5
11036        vpxor	xmm10, xmm10, xmm6
11037        vpclmulqdq	xmm9, xmm9, xmm10, 0
11038        vpxor	xmm9, xmm9, xmm8
11039        vpxor	xmm9, xmm9, xmm11
11040        vpslldq	xmm10, xmm9, 8
11041        vpsrldq	xmm9, xmm9, 8
11042        vpxor	xmm8, xmm8, xmm10
11043        vpxor	xmm6, xmm11, xmm9
11044        vpslld	xmm12, xmm8, 31
11045        vpslld	xmm13, xmm8, 30
11046        vpslld	xmm14, xmm8, 25
11047        vpxor	xmm12, xmm12, xmm13
11048        vpxor	xmm12, xmm12, xmm14
11049        vpsrldq	xmm13, xmm12, 4
11050        vpslldq	xmm12, xmm12, 12
11051        vpxor	xmm8, xmm8, xmm12
11052        vpsrld	xmm14, xmm8, 1
11053        vpsrld	xmm10, xmm8, 2
11054        vpsrld	xmm9, xmm8, 7
11055        vpxor	xmm14, xmm14, xmm10
11056        vpxor	xmm14, xmm14, xmm9
11057        vpxor	xmm14, xmm14, xmm13
11058        vpxor	xmm14, xmm14, xmm8
11059        vpxor	xmm6, xmm6, xmm14
11060L_AES_GCM_encrypt_update_avx1_last_block_done:
11061L_AES_GCM_encrypt_update_avx1_done_enc:
11062        vmovdqa	OWORD PTR [r12], xmm6
11063        vzeroupper
11064        vmovdqu	xmm6, OWORD PTR [rsp+160]
11065        vmovdqu	xmm7, OWORD PTR [rsp+176]
11066        vmovdqu	xmm8, OWORD PTR [rsp+192]
11067        vmovdqu	xmm9, OWORD PTR [rsp+208]
11068        vmovdqu	xmm10, OWORD PTR [rsp+224]
11069        vmovdqu	xmm11, OWORD PTR [rsp+240]
11070        vmovdqu	xmm12, OWORD PTR [rsp+256]
11071        vmovdqu	xmm13, OWORD PTR [rsp+272]
11072        vmovdqu	xmm14, OWORD PTR [rsp+288]
11073        vmovdqu	xmm15, OWORD PTR [rsp+304]
11074        add	rsp, 320
11075        pop	rdi
11076        pop	r15
11077        pop	r14
11078        pop	r12
11079        pop	r13
11080        ret
11081AES_GCM_encrypt_update_avx1 ENDP
11082_TEXT ENDS
11083_TEXT SEGMENT READONLY PARA
11084AES_GCM_encrypt_final_avx1 PROC
11085        push	r13
11086        push	r12
11087        push	r14
11088        mov	rax, rcx
11089        mov	r10d, r9d
11090        mov	r9, rdx
11091        mov	r11d, DWORD PTR [rsp+64]
11092        mov	r12, QWORD PTR [rsp+72]
11093        mov	r14, QWORD PTR [rsp+80]
11094        sub	rsp, 144
11095        vmovdqu	OWORD PTR [rsp+16], xmm6
11096        vmovdqu	OWORD PTR [rsp+32], xmm7
11097        vmovdqu	OWORD PTR [rsp+48], xmm8
11098        vmovdqu	OWORD PTR [rsp+64], xmm9
11099        vmovdqu	OWORD PTR [rsp+80], xmm10
11100        vmovdqu	OWORD PTR [rsp+96], xmm11
11101        vmovdqu	OWORD PTR [rsp+112], xmm12
11102        vmovdqu	OWORD PTR [rsp+128], xmm13
11103        vmovdqa	xmm4, OWORD PTR [rax]
11104        vmovdqa	xmm5, OWORD PTR [r12]
11105        vmovdqa	xmm6, OWORD PTR [r14]
11106        vpsrlq	xmm8, xmm5, 63
11107        vpsllq	xmm7, xmm5, 1
11108        vpslldq	xmm8, xmm8, 8
11109        vpor	xmm7, xmm7, xmm8
11110        vpshufd	xmm5, xmm5, 255
11111        vpsrad	xmm5, xmm5, 31
11112        vpand	xmm5, xmm5, OWORD PTR L_avx1_aes_gcm_mod2_128
11113        vpxor	xmm5, xmm5, xmm7
11114        mov	edx, r10d
11115        mov	ecx, r11d
11116        shl	rdx, 3
11117        shl	rcx, 3
11118        vmovq	xmm0, rdx
11119        vmovq	xmm1, rcx
11120        vpunpcklqdq	xmm0, xmm0, xmm1
11121        vpxor	xmm4, xmm4, xmm0
11122        ; ghash_gfmul_red_avx
11123        vpshufd	xmm8, xmm5, 78
11124        vpshufd	xmm9, xmm4, 78
11125        vpclmulqdq	xmm10, xmm4, xmm5, 17
11126        vpclmulqdq	xmm7, xmm4, xmm5, 0
11127        vpxor	xmm8, xmm8, xmm5
11128        vpxor	xmm9, xmm9, xmm4
11129        vpclmulqdq	xmm8, xmm8, xmm9, 0
11130        vpxor	xmm8, xmm8, xmm7
11131        vpxor	xmm8, xmm8, xmm10
11132        vpslldq	xmm9, xmm8, 8
11133        vpsrldq	xmm8, xmm8, 8
11134        vpxor	xmm7, xmm7, xmm9
11135        vpxor	xmm4, xmm10, xmm8
11136        vpslld	xmm11, xmm7, 31
11137        vpslld	xmm12, xmm7, 30
11138        vpslld	xmm13, xmm7, 25
11139        vpxor	xmm11, xmm11, xmm12
11140        vpxor	xmm11, xmm11, xmm13
11141        vpsrldq	xmm12, xmm11, 4
11142        vpslldq	xmm11, xmm11, 12
11143        vpxor	xmm7, xmm7, xmm11
11144        vpsrld	xmm13, xmm7, 1
11145        vpsrld	xmm9, xmm7, 2
11146        vpsrld	xmm8, xmm7, 7
11147        vpxor	xmm13, xmm13, xmm9
11148        vpxor	xmm13, xmm13, xmm8
11149        vpxor	xmm13, xmm13, xmm12
11150        vpxor	xmm13, xmm13, xmm7
11151        vpxor	xmm4, xmm4, xmm13
11152        vpshufb	xmm4, xmm4, OWORD PTR L_avx1_aes_gcm_bswap_mask
11153        vpxor	xmm0, xmm4, xmm6
11154        cmp	r8d, 16
11155        je	L_AES_GCM_encrypt_final_avx1_store_tag_16
11156        xor	rcx, rcx
11157        vmovdqu	OWORD PTR [rsp], xmm0
11158L_AES_GCM_encrypt_final_avx1_store_tag_loop:
11159        movzx	r13d, BYTE PTR [rsp+rcx]
11160        mov	BYTE PTR [r9+rcx], r13b
11161        inc	ecx
11162        cmp	ecx, r8d
11163        jne	L_AES_GCM_encrypt_final_avx1_store_tag_loop
11164        jmp	L_AES_GCM_encrypt_final_avx1_store_tag_done
11165L_AES_GCM_encrypt_final_avx1_store_tag_16:
11166        vmovdqu	OWORD PTR [r9], xmm0
11167L_AES_GCM_encrypt_final_avx1_store_tag_done:
11168        vzeroupper
11169        vmovdqu	xmm6, OWORD PTR [rsp+16]
11170        vmovdqu	xmm7, OWORD PTR [rsp+32]
11171        vmovdqu	xmm8, OWORD PTR [rsp+48]
11172        vmovdqu	xmm9, OWORD PTR [rsp+64]
11173        vmovdqu	xmm10, OWORD PTR [rsp+80]
11174        vmovdqu	xmm11, OWORD PTR [rsp+96]
11175        vmovdqu	xmm12, OWORD PTR [rsp+112]
11176        vmovdqu	xmm13, OWORD PTR [rsp+128]
11177        add	rsp, 144
11178        pop	r14
11179        pop	r12
11180        pop	r13
11181        ret
11182AES_GCM_encrypt_final_avx1 ENDP
11183_TEXT ENDS
11184_TEXT SEGMENT READONLY PARA
11185AES_GCM_decrypt_update_avx1 PROC
11186        push	r13
11187        push	r12
11188        push	r14
11189        push	r15
11190        push	rdi
11191        mov	rax, rcx
11192        mov	r10, r8
11193        mov	r8d, edx
11194        mov	r11, r9
11195        mov	r9d, DWORD PTR [rsp+80]
11196        mov	r12, QWORD PTR [rsp+88]
11197        mov	r14, QWORD PTR [rsp+96]
11198        mov	r15, QWORD PTR [rsp+104]
11199        sub	rsp, 328
11200        vmovdqu	OWORD PTR [rsp+168], xmm6
11201        vmovdqu	OWORD PTR [rsp+184], xmm7
11202        vmovdqu	OWORD PTR [rsp+200], xmm8
11203        vmovdqu	OWORD PTR [rsp+216], xmm9
11204        vmovdqu	OWORD PTR [rsp+232], xmm10
11205        vmovdqu	OWORD PTR [rsp+248], xmm11
11206        vmovdqu	OWORD PTR [rsp+264], xmm12
11207        vmovdqu	OWORD PTR [rsp+280], xmm13
11208        vmovdqu	OWORD PTR [rsp+296], xmm14
11209        vmovdqu	OWORD PTR [rsp+312], xmm15
11210        vmovdqa	xmm6, OWORD PTR [r12]
11211        vmovdqa	xmm5, OWORD PTR [r14]
11212        vpsrlq	xmm9, xmm5, 63
11213        vpsllq	xmm8, xmm5, 1
11214        vpslldq	xmm9, xmm9, 8
11215        vpor	xmm8, xmm8, xmm9
11216        vpshufd	xmm5, xmm5, 255
11217        vpsrad	xmm5, xmm5, 31
11218        vpand	xmm5, xmm5, OWORD PTR L_avx1_aes_gcm_mod2_128
11219        vpxor	xmm5, xmm5, xmm8
11220        xor	edi, edi
11221        cmp	r9d, 128
11222        mov	r13d, r9d
11223        jl	L_AES_GCM_decrypt_update_avx1_done_128
11224        and	r13d, 4294967168
11225        vmovdqa	xmm2, xmm6
11226        ; H ^ 1
11227        vmovdqu	OWORD PTR [rsp], xmm5
11228        ; H ^ 2
11229        vpclmulqdq	xmm8, xmm5, xmm5, 0
11230        vpclmulqdq	xmm0, xmm5, xmm5, 17
11231        vpslld	xmm12, xmm8, 31
11232        vpslld	xmm13, xmm8, 30
11233        vpslld	xmm14, xmm8, 25
11234        vpxor	xmm12, xmm12, xmm13
11235        vpxor	xmm12, xmm12, xmm14
11236        vpsrldq	xmm13, xmm12, 4
11237        vpslldq	xmm12, xmm12, 12
11238        vpxor	xmm8, xmm8, xmm12
11239        vpsrld	xmm14, xmm8, 1
11240        vpsrld	xmm10, xmm8, 2
11241        vpsrld	xmm9, xmm8, 7
11242        vpxor	xmm14, xmm14, xmm10
11243        vpxor	xmm14, xmm14, xmm9
11244        vpxor	xmm14, xmm14, xmm13
11245        vpxor	xmm14, xmm14, xmm8
11246        vpxor	xmm0, xmm0, xmm14
11247        vmovdqu	OWORD PTR [rsp+16], xmm0
11248        ; H ^ 3
11249        ; ghash_gfmul_red_avx
11250        vpshufd	xmm9, xmm5, 78
11251        vpshufd	xmm10, xmm0, 78
11252        vpclmulqdq	xmm11, xmm0, xmm5, 17
11253        vpclmulqdq	xmm8, xmm0, xmm5, 0
11254        vpxor	xmm9, xmm9, xmm5
11255        vpxor	xmm10, xmm10, xmm0
11256        vpclmulqdq	xmm9, xmm9, xmm10, 0
11257        vpxor	xmm9, xmm9, xmm8
11258        vpxor	xmm9, xmm9, xmm11
11259        vpslldq	xmm10, xmm9, 8
11260        vpsrldq	xmm9, xmm9, 8
11261        vpxor	xmm8, xmm8, xmm10
11262        vpxor	xmm1, xmm11, xmm9
11263        vpslld	xmm12, xmm8, 31
11264        vpslld	xmm13, xmm8, 30
11265        vpslld	xmm14, xmm8, 25
11266        vpxor	xmm12, xmm12, xmm13
11267        vpxor	xmm12, xmm12, xmm14
11268        vpsrldq	xmm13, xmm12, 4
11269        vpslldq	xmm12, xmm12, 12
11270        vpxor	xmm8, xmm8, xmm12
11271        vpsrld	xmm14, xmm8, 1
11272        vpsrld	xmm10, xmm8, 2
11273        vpsrld	xmm9, xmm8, 7
11274        vpxor	xmm14, xmm14, xmm10
11275        vpxor	xmm14, xmm14, xmm9
11276        vpxor	xmm14, xmm14, xmm13
11277        vpxor	xmm14, xmm14, xmm8
11278        vpxor	xmm1, xmm1, xmm14
11279        vmovdqu	OWORD PTR [rsp+32], xmm1
11280        ; H ^ 4
11281        vpclmulqdq	xmm8, xmm0, xmm0, 0
11282        vpclmulqdq	xmm3, xmm0, xmm0, 17
11283        vpslld	xmm12, xmm8, 31
11284        vpslld	xmm13, xmm8, 30
11285        vpslld	xmm14, xmm8, 25
11286        vpxor	xmm12, xmm12, xmm13
11287        vpxor	xmm12, xmm12, xmm14
11288        vpsrldq	xmm13, xmm12, 4
11289        vpslldq	xmm12, xmm12, 12
11290        vpxor	xmm8, xmm8, xmm12
11291        vpsrld	xmm14, xmm8, 1
11292        vpsrld	xmm10, xmm8, 2
11293        vpsrld	xmm9, xmm8, 7
11294        vpxor	xmm14, xmm14, xmm10
11295        vpxor	xmm14, xmm14, xmm9
11296        vpxor	xmm14, xmm14, xmm13
11297        vpxor	xmm14, xmm14, xmm8
11298        vpxor	xmm3, xmm3, xmm14
11299        vmovdqu	OWORD PTR [rsp+48], xmm3
11300        ; H ^ 5
11301        ; ghash_gfmul_red_avx
11302        vpshufd	xmm9, xmm0, 78
11303        vpshufd	xmm10, xmm1, 78
11304        vpclmulqdq	xmm11, xmm1, xmm0, 17
11305        vpclmulqdq	xmm8, xmm1, xmm0, 0
11306        vpxor	xmm9, xmm9, xmm0
11307        vpxor	xmm10, xmm10, xmm1
11308        vpclmulqdq	xmm9, xmm9, xmm10, 0
11309        vpxor	xmm9, xmm9, xmm8
11310        vpxor	xmm9, xmm9, xmm11
11311        vpslldq	xmm10, xmm9, 8
11312        vpsrldq	xmm9, xmm9, 8
11313        vpxor	xmm8, xmm8, xmm10
11314        vpxor	xmm7, xmm11, xmm9
11315        vpslld	xmm12, xmm8, 31
11316        vpslld	xmm13, xmm8, 30
11317        vpslld	xmm14, xmm8, 25
11318        vpxor	xmm12, xmm12, xmm13
11319        vpxor	xmm12, xmm12, xmm14
11320        vpsrldq	xmm13, xmm12, 4
11321        vpslldq	xmm12, xmm12, 12
11322        vpxor	xmm8, xmm8, xmm12
11323        vpsrld	xmm14, xmm8, 1
11324        vpsrld	xmm10, xmm8, 2
11325        vpsrld	xmm9, xmm8, 7
11326        vpxor	xmm14, xmm14, xmm10
11327        vpxor	xmm14, xmm14, xmm9
11328        vpxor	xmm14, xmm14, xmm13
11329        vpxor	xmm14, xmm14, xmm8
11330        vpxor	xmm7, xmm7, xmm14
11331        vmovdqu	OWORD PTR [rsp+64], xmm7
11332        ; H ^ 6
11333        vpclmulqdq	xmm8, xmm1, xmm1, 0
11334        vpclmulqdq	xmm7, xmm1, xmm1, 17
11335        vpslld	xmm12, xmm8, 31
11336        vpslld	xmm13, xmm8, 30
11337        vpslld	xmm14, xmm8, 25
11338        vpxor	xmm12, xmm12, xmm13
11339        vpxor	xmm12, xmm12, xmm14
11340        vpsrldq	xmm13, xmm12, 4
11341        vpslldq	xmm12, xmm12, 12
11342        vpxor	xmm8, xmm8, xmm12
11343        vpsrld	xmm14, xmm8, 1
11344        vpsrld	xmm10, xmm8, 2
11345        vpsrld	xmm9, xmm8, 7
11346        vpxor	xmm14, xmm14, xmm10
11347        vpxor	xmm14, xmm14, xmm9
11348        vpxor	xmm14, xmm14, xmm13
11349        vpxor	xmm14, xmm14, xmm8
11350        vpxor	xmm7, xmm7, xmm14
11351        vmovdqu	OWORD PTR [rsp+80], xmm7
11352        ; H ^ 7
11353        ; ghash_gfmul_red_avx
11354        vpshufd	xmm9, xmm1, 78
11355        vpshufd	xmm10, xmm3, 78
11356        vpclmulqdq	xmm11, xmm3, xmm1, 17
11357        vpclmulqdq	xmm8, xmm3, xmm1, 0
11358        vpxor	xmm9, xmm9, xmm1
11359        vpxor	xmm10, xmm10, xmm3
11360        vpclmulqdq	xmm9, xmm9, xmm10, 0
11361        vpxor	xmm9, xmm9, xmm8
11362        vpxor	xmm9, xmm9, xmm11
11363        vpslldq	xmm10, xmm9, 8
11364        vpsrldq	xmm9, xmm9, 8
11365        vpxor	xmm8, xmm8, xmm10
11366        vpxor	xmm7, xmm11, xmm9
11367        vpslld	xmm12, xmm8, 31
11368        vpslld	xmm13, xmm8, 30
11369        vpslld	xmm14, xmm8, 25
11370        vpxor	xmm12, xmm12, xmm13
11371        vpxor	xmm12, xmm12, xmm14
11372        vpsrldq	xmm13, xmm12, 4
11373        vpslldq	xmm12, xmm12, 12
11374        vpxor	xmm8, xmm8, xmm12
11375        vpsrld	xmm14, xmm8, 1
11376        vpsrld	xmm10, xmm8, 2
11377        vpsrld	xmm9, xmm8, 7
11378        vpxor	xmm14, xmm14, xmm10
11379        vpxor	xmm14, xmm14, xmm9
11380        vpxor	xmm14, xmm14, xmm13
11381        vpxor	xmm14, xmm14, xmm8
11382        vpxor	xmm7, xmm7, xmm14
11383        vmovdqu	OWORD PTR [rsp+96], xmm7
11384        ; H ^ 8
11385        vpclmulqdq	xmm8, xmm3, xmm3, 0
11386        vpclmulqdq	xmm7, xmm3, xmm3, 17
11387        vpslld	xmm12, xmm8, 31
11388        vpslld	xmm13, xmm8, 30
11389        vpslld	xmm14, xmm8, 25
11390        vpxor	xmm12, xmm12, xmm13
11391        vpxor	xmm12, xmm12, xmm14
11392        vpsrldq	xmm13, xmm12, 4
11393        vpslldq	xmm12, xmm12, 12
11394        vpxor	xmm8, xmm8, xmm12
11395        vpsrld	xmm14, xmm8, 1
11396        vpsrld	xmm10, xmm8, 2
11397        vpsrld	xmm9, xmm8, 7
11398        vpxor	xmm14, xmm14, xmm10
11399        vpxor	xmm14, xmm14, xmm9
11400        vpxor	xmm14, xmm14, xmm13
11401        vpxor	xmm14, xmm14, xmm8
11402        vpxor	xmm7, xmm7, xmm14
11403        vmovdqu	OWORD PTR [rsp+112], xmm7
11404L_AES_GCM_decrypt_update_avx1_ghash_128:
11405        lea	rcx, QWORD PTR [r11+rdi]
11406        lea	rdx, QWORD PTR [r10+rdi]
11407        vmovdqu	xmm0, OWORD PTR [r15]
11408        vmovdqa	xmm1, OWORD PTR L_avx1_aes_gcm_bswap_epi64
11409        vpshufb	xmm8, xmm0, xmm1
11410        vpaddd	xmm9, xmm0, OWORD PTR L_avx1_aes_gcm_one
11411        vpshufb	xmm9, xmm9, xmm1
11412        vpaddd	xmm10, xmm0, OWORD PTR L_avx1_aes_gcm_two
11413        vpshufb	xmm10, xmm10, xmm1
11414        vpaddd	xmm11, xmm0, OWORD PTR L_avx1_aes_gcm_three
11415        vpshufb	xmm11, xmm11, xmm1
11416        vpaddd	xmm12, xmm0, OWORD PTR L_avx1_aes_gcm_four
11417        vpshufb	xmm12, xmm12, xmm1
11418        vpaddd	xmm13, xmm0, OWORD PTR L_avx1_aes_gcm_five
11419        vpshufb	xmm13, xmm13, xmm1
11420        vpaddd	xmm14, xmm0, OWORD PTR L_avx1_aes_gcm_six
11421        vpshufb	xmm14, xmm14, xmm1
11422        vpaddd	xmm15, xmm0, OWORD PTR L_avx1_aes_gcm_seven
11423        vpshufb	xmm15, xmm15, xmm1
11424        vpaddd	xmm0, xmm0, OWORD PTR L_avx1_aes_gcm_eight
11425        vmovdqa	xmm7, OWORD PTR [rax]
11426        vmovdqu	OWORD PTR [r15], xmm0
11427        vpxor	xmm8, xmm8, xmm7
11428        vpxor	xmm9, xmm9, xmm7
11429        vpxor	xmm10, xmm10, xmm7
11430        vpxor	xmm11, xmm11, xmm7
11431        vpxor	xmm12, xmm12, xmm7
11432        vpxor	xmm13, xmm13, xmm7
11433        vpxor	xmm14, xmm14, xmm7
11434        vpxor	xmm15, xmm15, xmm7
11435        vmovdqu	xmm7, OWORD PTR [rsp+112]
11436        vmovdqu	xmm0, OWORD PTR [rcx]
11437        vaesenc	xmm8, xmm8, [rax+16]
11438        vpshufb	xmm0, xmm0, OWORD PTR L_avx1_aes_gcm_bswap_mask
11439        vpxor	xmm0, xmm0, xmm2
11440        vpshufd	xmm1, xmm7, 78
11441        vpshufd	xmm5, xmm0, 78
11442        vpxor	xmm1, xmm1, xmm7
11443        vpxor	xmm5, xmm5, xmm0
11444        vpclmulqdq	xmm3, xmm0, xmm7, 17
11445        vaesenc	xmm9, xmm9, [rax+16]
11446        vaesenc	xmm10, xmm10, [rax+16]
11447        vpclmulqdq	xmm2, xmm0, xmm7, 0
11448        vaesenc	xmm11, xmm11, [rax+16]
11449        vaesenc	xmm12, xmm12, [rax+16]
11450        vpclmulqdq	xmm1, xmm1, xmm5, 0
11451        vaesenc	xmm13, xmm13, [rax+16]
11452        vaesenc	xmm14, xmm14, [rax+16]
11453        vaesenc	xmm15, xmm15, [rax+16]
11454        vpxor	xmm1, xmm1, xmm2
11455        vpxor	xmm1, xmm1, xmm3
11456        vmovdqu	xmm7, OWORD PTR [rsp+96]
11457        vmovdqu	xmm0, OWORD PTR [rcx+16]
11458        vpshufd	xmm4, xmm7, 78
11459        vpshufb	xmm0, xmm0, OWORD PTR L_avx1_aes_gcm_bswap_mask
11460        vaesenc	xmm8, xmm8, [rax+32]
11461        vpxor	xmm4, xmm4, xmm7
11462        vpshufd	xmm5, xmm0, 78
11463        vpxor	xmm5, xmm5, xmm0
11464        vpclmulqdq	xmm6, xmm0, xmm7, 17
11465        vaesenc	xmm9, xmm9, [rax+32]
11466        vaesenc	xmm10, xmm10, [rax+32]
11467        vpclmulqdq	xmm7, xmm0, xmm7, 0
11468        vaesenc	xmm11, xmm11, [rax+32]
11469        vaesenc	xmm12, xmm12, [rax+32]
11470        vpclmulqdq	xmm4, xmm4, xmm5, 0
11471        vaesenc	xmm13, xmm13, [rax+32]
11472        vaesenc	xmm14, xmm14, [rax+32]
11473        vaesenc	xmm15, xmm15, [rax+32]
11474        vpxor	xmm1, xmm1, xmm7
11475        vpxor	xmm2, xmm2, xmm7
11476        vpxor	xmm1, xmm1, xmm6
11477        vpxor	xmm3, xmm3, xmm6
11478        vpxor	xmm1, xmm1, xmm4
11479        vmovdqu	xmm7, OWORD PTR [rsp+80]
11480        vmovdqu	xmm0, OWORD PTR [rcx+32]
11481        vpshufd	xmm4, xmm7, 78
11482        vpshufb	xmm0, xmm0, OWORD PTR L_avx1_aes_gcm_bswap_mask
11483        vaesenc	xmm8, xmm8, [rax+48]
11484        vpxor	xmm4, xmm4, xmm7
11485        vpshufd	xmm5, xmm0, 78
11486        vpxor	xmm5, xmm5, xmm0
11487        vpclmulqdq	xmm6, xmm0, xmm7, 17
11488        vaesenc	xmm9, xmm9, [rax+48]
11489        vaesenc	xmm10, xmm10, [rax+48]
11490        vpclmulqdq	xmm7, xmm0, xmm7, 0
11491        vaesenc	xmm11, xmm11, [rax+48]
11492        vaesenc	xmm12, xmm12, [rax+48]
11493        vpclmulqdq	xmm4, xmm4, xmm5, 0
11494        vaesenc	xmm13, xmm13, [rax+48]
11495        vaesenc	xmm14, xmm14, [rax+48]
11496        vaesenc	xmm15, xmm15, [rax+48]
11497        vpxor	xmm1, xmm1, xmm7
11498        vpxor	xmm2, xmm2, xmm7
11499        vpxor	xmm1, xmm1, xmm6
11500        vpxor	xmm3, xmm3, xmm6
11501        vpxor	xmm1, xmm1, xmm4
11502        vmovdqu	xmm7, OWORD PTR [rsp+64]
11503        vmovdqu	xmm0, OWORD PTR [rcx+48]
11504        vpshufd	xmm4, xmm7, 78
11505        vpshufb	xmm0, xmm0, OWORD PTR L_avx1_aes_gcm_bswap_mask
11506        vaesenc	xmm8, xmm8, [rax+64]
11507        vpxor	xmm4, xmm4, xmm7
11508        vpshufd	xmm5, xmm0, 78
11509        vpxor	xmm5, xmm5, xmm0
11510        vpclmulqdq	xmm6, xmm0, xmm7, 17
11511        vaesenc	xmm9, xmm9, [rax+64]
11512        vaesenc	xmm10, xmm10, [rax+64]
11513        vpclmulqdq	xmm7, xmm0, xmm7, 0
11514        vaesenc	xmm11, xmm11, [rax+64]
11515        vaesenc	xmm12, xmm12, [rax+64]
11516        vpclmulqdq	xmm4, xmm4, xmm5, 0
11517        vaesenc	xmm13, xmm13, [rax+64]
11518        vaesenc	xmm14, xmm14, [rax+64]
11519        vaesenc	xmm15, xmm15, [rax+64]
11520        vpxor	xmm1, xmm1, xmm7
11521        vpxor	xmm2, xmm2, xmm7
11522        vpxor	xmm1, xmm1, xmm6
11523        vpxor	xmm3, xmm3, xmm6
11524        vpxor	xmm1, xmm1, xmm4
11525        vmovdqu	xmm7, OWORD PTR [rsp+48]
11526        vmovdqu	xmm0, OWORD PTR [rcx+64]
11527        vpshufd	xmm4, xmm7, 78
11528        vpshufb	xmm0, xmm0, OWORD PTR L_avx1_aes_gcm_bswap_mask
11529        vaesenc	xmm8, xmm8, [rax+80]
11530        vpxor	xmm4, xmm4, xmm7
11531        vpshufd	xmm5, xmm0, 78
11532        vpxor	xmm5, xmm5, xmm0
11533        vpclmulqdq	xmm6, xmm0, xmm7, 17
11534        vaesenc	xmm9, xmm9, [rax+80]
11535        vaesenc	xmm10, xmm10, [rax+80]
11536        vpclmulqdq	xmm7, xmm0, xmm7, 0
11537        vaesenc	xmm11, xmm11, [rax+80]
11538        vaesenc	xmm12, xmm12, [rax+80]
11539        vpclmulqdq	xmm4, xmm4, xmm5, 0
11540        vaesenc	xmm13, xmm13, [rax+80]
11541        vaesenc	xmm14, xmm14, [rax+80]
11542        vaesenc	xmm15, xmm15, [rax+80]
11543        vpxor	xmm1, xmm1, xmm7
11544        vpxor	xmm2, xmm2, xmm7
11545        vpxor	xmm1, xmm1, xmm6
11546        vpxor	xmm3, xmm3, xmm6
11547        vpxor	xmm1, xmm1, xmm4
11548        vmovdqu	xmm7, OWORD PTR [rsp+32]
11549        vmovdqu	xmm0, OWORD PTR [rcx+80]
11550        vpshufd	xmm4, xmm7, 78
11551        vpshufb	xmm0, xmm0, OWORD PTR L_avx1_aes_gcm_bswap_mask
11552        vaesenc	xmm8, xmm8, [rax+96]
11553        vpxor	xmm4, xmm4, xmm7
11554        vpshufd	xmm5, xmm0, 78
11555        vpxor	xmm5, xmm5, xmm0
11556        vpclmulqdq	xmm6, xmm0, xmm7, 17
11557        vaesenc	xmm9, xmm9, [rax+96]
11558        vaesenc	xmm10, xmm10, [rax+96]
11559        vpclmulqdq	xmm7, xmm0, xmm7, 0
11560        vaesenc	xmm11, xmm11, [rax+96]
11561        vaesenc	xmm12, xmm12, [rax+96]
11562        vpclmulqdq	xmm4, xmm4, xmm5, 0
11563        vaesenc	xmm13, xmm13, [rax+96]
11564        vaesenc	xmm14, xmm14, [rax+96]
11565        vaesenc	xmm15, xmm15, [rax+96]
11566        vpxor	xmm1, xmm1, xmm7
11567        vpxor	xmm2, xmm2, xmm7
11568        vpxor	xmm1, xmm1, xmm6
11569        vpxor	xmm3, xmm3, xmm6
11570        vpxor	xmm1, xmm1, xmm4
11571        vmovdqu	xmm7, OWORD PTR [rsp+16]
11572        vmovdqu	xmm0, OWORD PTR [rcx+96]
11573        vpshufd	xmm4, xmm7, 78
11574        vpshufb	xmm0, xmm0, OWORD PTR L_avx1_aes_gcm_bswap_mask
11575        vaesenc	xmm8, xmm8, [rax+112]
11576        vpxor	xmm4, xmm4, xmm7
11577        vpshufd	xmm5, xmm0, 78
11578        vpxor	xmm5, xmm5, xmm0
11579        vpclmulqdq	xmm6, xmm0, xmm7, 17
11580        vaesenc	xmm9, xmm9, [rax+112]
11581        vaesenc	xmm10, xmm10, [rax+112]
11582        vpclmulqdq	xmm7, xmm0, xmm7, 0
11583        vaesenc	xmm11, xmm11, [rax+112]
11584        vaesenc	xmm12, xmm12, [rax+112]
11585        vpclmulqdq	xmm4, xmm4, xmm5, 0
11586        vaesenc	xmm13, xmm13, [rax+112]
11587        vaesenc	xmm14, xmm14, [rax+112]
11588        vaesenc	xmm15, xmm15, [rax+112]
11589        vpxor	xmm1, xmm1, xmm7
11590        vpxor	xmm2, xmm2, xmm7
11591        vpxor	xmm1, xmm1, xmm6
11592        vpxor	xmm3, xmm3, xmm6
11593        vpxor	xmm1, xmm1, xmm4
11594        vmovdqu	xmm7, OWORD PTR [rsp]
11595        vmovdqu	xmm0, OWORD PTR [rcx+112]
11596        vpshufd	xmm4, xmm7, 78
11597        vpshufb	xmm0, xmm0, OWORD PTR L_avx1_aes_gcm_bswap_mask
11598        vaesenc	xmm8, xmm8, [rax+128]
11599        vpxor	xmm4, xmm4, xmm7
11600        vpshufd	xmm5, xmm0, 78
11601        vpxor	xmm5, xmm5, xmm0
11602        vpclmulqdq	xmm6, xmm0, xmm7, 17
11603        vaesenc	xmm9, xmm9, [rax+128]
11604        vaesenc	xmm10, xmm10, [rax+128]
11605        vpclmulqdq	xmm7, xmm0, xmm7, 0
11606        vaesenc	xmm11, xmm11, [rax+128]
11607        vaesenc	xmm12, xmm12, [rax+128]
11608        vpclmulqdq	xmm4, xmm4, xmm5, 0
11609        vaesenc	xmm13, xmm13, [rax+128]
11610        vaesenc	xmm14, xmm14, [rax+128]
11611        vaesenc	xmm15, xmm15, [rax+128]
11612        vpxor	xmm1, xmm1, xmm7
11613        vpxor	xmm2, xmm2, xmm7
11614        vpxor	xmm1, xmm1, xmm6
11615        vpxor	xmm3, xmm3, xmm6
11616        vpxor	xmm1, xmm1, xmm4
11617        vpslldq	xmm5, xmm1, 8
11618        vpsrldq	xmm1, xmm1, 8
11619        vaesenc	xmm8, xmm8, [rax+144]
11620        vpxor	xmm2, xmm2, xmm5
11621        vpxor	xmm3, xmm3, xmm1
11622        vaesenc	xmm9, xmm9, [rax+144]
11623        vpslld	xmm7, xmm2, 31
11624        vpslld	xmm4, xmm2, 30
11625        vpslld	xmm5, xmm2, 25
11626        vaesenc	xmm10, xmm10, [rax+144]
11627        vpxor	xmm7, xmm7, xmm4
11628        vpxor	xmm7, xmm7, xmm5
11629        vaesenc	xmm11, xmm11, [rax+144]
11630        vpsrldq	xmm4, xmm7, 4
11631        vpslldq	xmm7, xmm7, 12
11632        vaesenc	xmm12, xmm12, [rax+144]
11633        vpxor	xmm2, xmm2, xmm7
11634        vpsrld	xmm5, xmm2, 1
11635        vaesenc	xmm13, xmm13, [rax+144]
11636        vpsrld	xmm1, xmm2, 2
11637        vpsrld	xmm0, xmm2, 7
11638        vaesenc	xmm14, xmm14, [rax+144]
11639        vpxor	xmm5, xmm5, xmm1
11640        vpxor	xmm5, xmm5, xmm0
11641        vaesenc	xmm15, xmm15, [rax+144]
11642        vpxor	xmm5, xmm5, xmm4
11643        vpxor	xmm2, xmm2, xmm5
11644        vpxor	xmm2, xmm2, xmm3
11645        cmp	r8d, 11
11646        vmovdqa	xmm7, OWORD PTR [rax+160]
11647        jl	L_AES_GCM_decrypt_update_avx1_aesenc_128_ghash_avx_done
11648        vaesenc	xmm8, xmm8, xmm7
11649        vaesenc	xmm9, xmm9, xmm7
11650        vaesenc	xmm10, xmm10, xmm7
11651        vaesenc	xmm11, xmm11, xmm7
11652        vaesenc	xmm12, xmm12, xmm7
11653        vaesenc	xmm13, xmm13, xmm7
11654        vaesenc	xmm14, xmm14, xmm7
11655        vaesenc	xmm15, xmm15, xmm7
11656        vmovdqa	xmm7, OWORD PTR [rax+176]
11657        vaesenc	xmm8, xmm8, xmm7
11658        vaesenc	xmm9, xmm9, xmm7
11659        vaesenc	xmm10, xmm10, xmm7
11660        vaesenc	xmm11, xmm11, xmm7
11661        vaesenc	xmm12, xmm12, xmm7
11662        vaesenc	xmm13, xmm13, xmm7
11663        vaesenc	xmm14, xmm14, xmm7
11664        vaesenc	xmm15, xmm15, xmm7
11665        cmp	r8d, 13
11666        vmovdqa	xmm7, OWORD PTR [rax+192]
11667        jl	L_AES_GCM_decrypt_update_avx1_aesenc_128_ghash_avx_done
11668        vaesenc	xmm8, xmm8, xmm7
11669        vaesenc	xmm9, xmm9, xmm7
11670        vaesenc	xmm10, xmm10, xmm7
11671        vaesenc	xmm11, xmm11, xmm7
11672        vaesenc	xmm12, xmm12, xmm7
11673        vaesenc	xmm13, xmm13, xmm7
11674        vaesenc	xmm14, xmm14, xmm7
11675        vaesenc	xmm15, xmm15, xmm7
11676        vmovdqa	xmm7, OWORD PTR [rax+208]
11677        vaesenc	xmm8, xmm8, xmm7
11678        vaesenc	xmm9, xmm9, xmm7
11679        vaesenc	xmm10, xmm10, xmm7
11680        vaesenc	xmm11, xmm11, xmm7
11681        vaesenc	xmm12, xmm12, xmm7
11682        vaesenc	xmm13, xmm13, xmm7
11683        vaesenc	xmm14, xmm14, xmm7
11684        vaesenc	xmm15, xmm15, xmm7
11685        vmovdqa	xmm7, OWORD PTR [rax+224]
11686L_AES_GCM_decrypt_update_avx1_aesenc_128_ghash_avx_done:
11687        vaesenclast	xmm8, xmm8, xmm7
11688        vaesenclast	xmm9, xmm9, xmm7
11689        vmovdqu	xmm0, OWORD PTR [rcx]
11690        vmovdqu	xmm1, OWORD PTR [rcx+16]
11691        vpxor	xmm8, xmm8, xmm0
11692        vpxor	xmm9, xmm9, xmm1
11693        vmovdqu	OWORD PTR [rdx], xmm8
11694        vmovdqu	OWORD PTR [rdx+16], xmm9
11695        vaesenclast	xmm10, xmm10, xmm7
11696        vaesenclast	xmm11, xmm11, xmm7
11697        vmovdqu	xmm0, OWORD PTR [rcx+32]
11698        vmovdqu	xmm1, OWORD PTR [rcx+48]
11699        vpxor	xmm10, xmm10, xmm0
11700        vpxor	xmm11, xmm11, xmm1
11701        vmovdqu	OWORD PTR [rdx+32], xmm10
11702        vmovdqu	OWORD PTR [rdx+48], xmm11
11703        vaesenclast	xmm12, xmm12, xmm7
11704        vaesenclast	xmm13, xmm13, xmm7
11705        vmovdqu	xmm0, OWORD PTR [rcx+64]
11706        vmovdqu	xmm1, OWORD PTR [rcx+80]
11707        vpxor	xmm12, xmm12, xmm0
11708        vpxor	xmm13, xmm13, xmm1
11709        vmovdqu	OWORD PTR [rdx+64], xmm12
11710        vmovdqu	OWORD PTR [rdx+80], xmm13
11711        vaesenclast	xmm14, xmm14, xmm7
11712        vaesenclast	xmm15, xmm15, xmm7
11713        vmovdqu	xmm0, OWORD PTR [rcx+96]
11714        vmovdqu	xmm1, OWORD PTR [rcx+112]
11715        vpxor	xmm14, xmm14, xmm0
11716        vpxor	xmm15, xmm15, xmm1
11717        vmovdqu	OWORD PTR [rdx+96], xmm14
11718        vmovdqu	OWORD PTR [rdx+112], xmm15
11719        add	edi, 128
11720        cmp	edi, r13d
11721        jl	L_AES_GCM_decrypt_update_avx1_ghash_128
11722        vmovdqa	xmm6, xmm2
11723        vmovdqu	xmm5, OWORD PTR [rsp]
11724L_AES_GCM_decrypt_update_avx1_done_128:
11725        mov	edx, r9d
11726        cmp	edi, edx
11727        jge	L_AES_GCM_decrypt_update_avx1_done_dec
11728        mov	r13d, r9d
11729        and	r13d, 4294967280
11730        cmp	edi, r13d
11731        jge	L_AES_GCM_decrypt_update_avx1_last_block_done
11732L_AES_GCM_decrypt_update_avx1_last_block_start:
11733        vmovdqu	xmm13, OWORD PTR [r11+rdi]
11734        vmovdqa	xmm0, xmm5
11735        vpshufb	xmm1, xmm13, OWORD PTR L_avx1_aes_gcm_bswap_mask
11736        vpxor	xmm1, xmm1, xmm6
11737        vmovdqu	xmm9, OWORD PTR [r15]
11738        vpshufb	xmm8, xmm9, OWORD PTR L_avx1_aes_gcm_bswap_epi64
11739        vpaddd	xmm9, xmm9, OWORD PTR L_avx1_aes_gcm_one
11740        vmovdqu	OWORD PTR [r15], xmm9
11741        vpxor	xmm8, xmm8, [rax]
11742        vpclmulqdq	xmm10, xmm1, xmm0, 16
11743        vaesenc	xmm8, xmm8, [rax+16]
11744        vaesenc	xmm8, xmm8, [rax+32]
11745        vpclmulqdq	xmm11, xmm1, xmm0, 1
11746        vaesenc	xmm8, xmm8, [rax+48]
11747        vaesenc	xmm8, xmm8, [rax+64]
11748        vpclmulqdq	xmm12, xmm1, xmm0, 0
11749        vaesenc	xmm8, xmm8, [rax+80]
11750        vpclmulqdq	xmm1, xmm1, xmm0, 17
11751        vaesenc	xmm8, xmm8, [rax+96]
11752        vpxor	xmm10, xmm10, xmm11
11753        vpslldq	xmm2, xmm10, 8
11754        vpsrldq	xmm10, xmm10, 8
11755        vaesenc	xmm8, xmm8, [rax+112]
11756        vpxor	xmm2, xmm2, xmm12
11757        vpxor	xmm3, xmm1, xmm10
11758        vmovdqa	xmm0, OWORD PTR L_avx1_aes_gcm_mod2_128
11759        vpclmulqdq	xmm11, xmm2, xmm0, 16
11760        vaesenc	xmm8, xmm8, [rax+128]
11761        vpshufd	xmm10, xmm2, 78
11762        vpxor	xmm10, xmm10, xmm11
11763        vpclmulqdq	xmm11, xmm10, xmm0, 16
11764        vaesenc	xmm8, xmm8, [rax+144]
11765        vpshufd	xmm10, xmm10, 78
11766        vpxor	xmm10, xmm10, xmm11
11767        vpxor	xmm6, xmm10, xmm3
11768        cmp	r8d, 11
11769        vmovdqa	xmm9, OWORD PTR [rax+160]
11770        jl	L_AES_GCM_decrypt_update_avx1_aesenc_gfmul_last
11771        vaesenc	xmm8, xmm8, xmm9
11772        vaesenc	xmm8, xmm8, [rax+176]
11773        cmp	r8d, 13
11774        vmovdqa	xmm9, OWORD PTR [rax+192]
11775        jl	L_AES_GCM_decrypt_update_avx1_aesenc_gfmul_last
11776        vaesenc	xmm8, xmm8, xmm9
11777        vaesenc	xmm8, xmm8, [rax+208]
11778        vmovdqa	xmm9, OWORD PTR [rax+224]
11779L_AES_GCM_decrypt_update_avx1_aesenc_gfmul_last:
11780        vaesenclast	xmm8, xmm8, xmm9
11781        vmovdqa	xmm0, xmm13
11782        vpxor	xmm8, xmm8, xmm0
11783        vmovdqu	OWORD PTR [r10+rdi], xmm8
11784        add	edi, 16
11785        cmp	edi, r13d
11786        jl	L_AES_GCM_decrypt_update_avx1_last_block_start
11787L_AES_GCM_decrypt_update_avx1_last_block_done:
11788L_AES_GCM_decrypt_update_avx1_done_dec:
11789        vmovdqa	OWORD PTR [r12], xmm6
11790        vzeroupper
11791        vmovdqu	xmm6, OWORD PTR [rsp+168]
11792        vmovdqu	xmm7, OWORD PTR [rsp+184]
11793        vmovdqu	xmm8, OWORD PTR [rsp+200]
11794        vmovdqu	xmm9, OWORD PTR [rsp+216]
11795        vmovdqu	xmm10, OWORD PTR [rsp+232]
11796        vmovdqu	xmm11, OWORD PTR [rsp+248]
11797        vmovdqu	xmm12, OWORD PTR [rsp+264]
11798        vmovdqu	xmm13, OWORD PTR [rsp+280]
11799        vmovdqu	xmm14, OWORD PTR [rsp+296]
11800        vmovdqu	xmm15, OWORD PTR [rsp+312]
11801        add	rsp, 328
11802        pop	rdi
11803        pop	r15
11804        pop	r14
11805        pop	r12
11806        pop	r13
11807        ret
11808AES_GCM_decrypt_update_avx1 ENDP
11809_TEXT ENDS
11810_TEXT SEGMENT READONLY PARA
11811AES_GCM_decrypt_final_avx1 PROC
11812        push	r13
11813        push	r12
11814        push	r14
11815        push	rbp
11816        push	r15
11817        mov	rax, rcx
11818        mov	r10d, r9d
11819        mov	r9, rdx
11820        mov	r11d, DWORD PTR [rsp+80]
11821        mov	r12, QWORD PTR [rsp+88]
11822        mov	r14, QWORD PTR [rsp+96]
11823        mov	rbp, QWORD PTR [rsp+104]
11824        sub	rsp, 160
11825        vmovdqu	OWORD PTR [rsp+16], xmm6
11826        vmovdqu	OWORD PTR [rsp+32], xmm7
11827        vmovdqu	OWORD PTR [rsp+48], xmm8
11828        vmovdqu	OWORD PTR [rsp+64], xmm9
11829        vmovdqu	OWORD PTR [rsp+80], xmm10
11830        vmovdqu	OWORD PTR [rsp+96], xmm11
11831        vmovdqu	OWORD PTR [rsp+112], xmm12
11832        vmovdqu	OWORD PTR [rsp+128], xmm13
11833        vmovdqu	OWORD PTR [rsp+144], xmm15
11834        vmovdqa	xmm6, OWORD PTR [rax]
11835        vmovdqa	xmm5, OWORD PTR [r12]
11836        vmovdqa	xmm15, OWORD PTR [r14]
11837        vpsrlq	xmm8, xmm5, 63
11838        vpsllq	xmm7, xmm5, 1
11839        vpslldq	xmm8, xmm8, 8
11840        vpor	xmm7, xmm7, xmm8
11841        vpshufd	xmm5, xmm5, 255
11842        vpsrad	xmm5, xmm5, 31
11843        vpand	xmm5, xmm5, OWORD PTR L_avx1_aes_gcm_mod2_128
11844        vpxor	xmm5, xmm5, xmm7
11845        mov	edx, r10d
11846        mov	ecx, r11d
11847        shl	rdx, 3
11848        shl	rcx, 3
11849        vmovq	xmm0, rdx
11850        vmovq	xmm1, rcx
11851        vpunpcklqdq	xmm0, xmm0, xmm1
11852        vpxor	xmm6, xmm6, xmm0
11853        ; ghash_gfmul_red_avx
11854        vpshufd	xmm8, xmm5, 78
11855        vpshufd	xmm9, xmm6, 78
11856        vpclmulqdq	xmm10, xmm6, xmm5, 17
11857        vpclmulqdq	xmm7, xmm6, xmm5, 0
11858        vpxor	xmm8, xmm8, xmm5
11859        vpxor	xmm9, xmm9, xmm6
11860        vpclmulqdq	xmm8, xmm8, xmm9, 0
11861        vpxor	xmm8, xmm8, xmm7
11862        vpxor	xmm8, xmm8, xmm10
11863        vpslldq	xmm9, xmm8, 8
11864        vpsrldq	xmm8, xmm8, 8
11865        vpxor	xmm7, xmm7, xmm9
11866        vpxor	xmm6, xmm10, xmm8
11867        vpslld	xmm11, xmm7, 31
11868        vpslld	xmm12, xmm7, 30
11869        vpslld	xmm13, xmm7, 25
11870        vpxor	xmm11, xmm11, xmm12
11871        vpxor	xmm11, xmm11, xmm13
11872        vpsrldq	xmm12, xmm11, 4
11873        vpslldq	xmm11, xmm11, 12
11874        vpxor	xmm7, xmm7, xmm11
11875        vpsrld	xmm13, xmm7, 1
11876        vpsrld	xmm9, xmm7, 2
11877        vpsrld	xmm8, xmm7, 7
11878        vpxor	xmm13, xmm13, xmm9
11879        vpxor	xmm13, xmm13, xmm8
11880        vpxor	xmm13, xmm13, xmm12
11881        vpxor	xmm13, xmm13, xmm7
11882        vpxor	xmm6, xmm6, xmm13
11883        vpshufb	xmm6, xmm6, OWORD PTR L_avx1_aes_gcm_bswap_mask
11884        vpxor	xmm0, xmm6, xmm15
11885        cmp	r8d, 16
11886        je	L_AES_GCM_decrypt_final_avx1_cmp_tag_16
11887        sub	rsp, 16
11888        xor	rcx, rcx
11889        xor	r15, r15
11890        vmovdqu	OWORD PTR [rsp], xmm0
11891L_AES_GCM_decrypt_final_avx1_cmp_tag_loop:
11892        movzx	r13d, BYTE PTR [rsp+rcx]
11893        xor	r13b, BYTE PTR [r9+rcx]
11894        or	r15b, r13b
11895        inc	ecx
11896        cmp	ecx, r8d
11897        jne	L_AES_GCM_decrypt_final_avx1_cmp_tag_loop
11898        cmp	r15b, 0
11899        sete	r15b
11900        add	rsp, 16
11901        xor	rcx, rcx
11902        jmp	L_AES_GCM_decrypt_final_avx1_cmp_tag_done
11903L_AES_GCM_decrypt_final_avx1_cmp_tag_16:
11904        vmovdqu	xmm1, OWORD PTR [r9]
11905        vpcmpeqb	xmm0, xmm0, xmm1
11906        vpmovmskb	rdx, xmm0
11907        ; %%edx == 0xFFFF then return 1 else => return 0
11908        xor	r15d, r15d
11909        cmp	edx, 65535
11910        sete	r15b
11911L_AES_GCM_decrypt_final_avx1_cmp_tag_done:
11912        mov	DWORD PTR [rbp], r15d
11913        vzeroupper
11914        vmovdqu	xmm6, OWORD PTR [rsp+16]
11915        vmovdqu	xmm7, OWORD PTR [rsp+32]
11916        vmovdqu	xmm8, OWORD PTR [rsp+48]
11917        vmovdqu	xmm9, OWORD PTR [rsp+64]
11918        vmovdqu	xmm10, OWORD PTR [rsp+80]
11919        vmovdqu	xmm11, OWORD PTR [rsp+96]
11920        vmovdqu	xmm12, OWORD PTR [rsp+112]
11921        vmovdqu	xmm13, OWORD PTR [rsp+128]
11922        vmovdqu	xmm15, OWORD PTR [rsp+144]
11923        add	rsp, 160
11924        pop	r15
11925        pop	rbp
11926        pop	r14
11927        pop	r12
11928        pop	r13
11929        ret
11930AES_GCM_decrypt_final_avx1 ENDP
11931_TEXT ENDS
11932ENDIF
11933IFDEF HAVE_INTEL_AVX2
11934_DATA SEGMENT
11935ALIGN 16
11936L_GCM_generate_m0_avx2_rev8 QWORD \
11937     08090a0b0c0d0e0fh,  0001020304050607h
11938ptr_L_GCM_generate_m0_avx2_rev8 QWORD L_GCM_generate_m0_avx2_rev8
11939_DATA ENDS
11940_DATA SEGMENT
11941ALIGN 16
11942L_GCM_generate_m0_avx2_mod2_128 QWORD \
11943     0000000000000000h, 0e100000000000000h
11944ptr_L_GCM_generate_m0_avx2_mod2_128 QWORD L_GCM_generate_m0_avx2_mod2_128
11945_DATA ENDS
11946_TEXT SEGMENT READONLY PARA
11947GCM_generate_m0_avx2 PROC
11948        sub	rsp, 80
11949        vmovdqu	OWORD PTR [rsp], xmm6
11950        vmovdqu	OWORD PTR [rsp+16], xmm7
11951        vmovdqu	OWORD PTR [rsp+32], xmm8
11952        vmovdqu	OWORD PTR [rsp+48], xmm9
11953        vmovdqu	OWORD PTR [rsp+64], xmm10
11954        vmovdqu	xmm9, OWORD PTR L_GCM_generate_m0_avx2_rev8
11955        vmovdqu	xmm10, OWORD PTR L_GCM_generate_m0_avx2_mod2_128
11956        vpxor	xmm8, xmm8, xmm8
11957        vmovdqu	xmm0, OWORD PTR [rcx]
11958        vmovdqu	OWORD PTR [rdx], xmm8
11959        vmovdqu	xmm8, xmm0
11960        vpshufb	xmm0, xmm0, xmm9
11961        vpsllq	xmm5, xmm0, 63
11962        vpsrlq	xmm4, xmm0, 1
11963        vpslldq	xmm1, xmm5, 8
11964        vpsrldq	xmm5, xmm5, 8
11965        vpshufd	xmm1, xmm1, 255
11966        vpor	xmm4, xmm4, xmm5
11967        vpsrad	xmm1, xmm1, 31
11968        vpand	xmm1, xmm1, xmm10
11969        vpxor	xmm1, xmm1, xmm4
11970        vpsllq	xmm5, xmm1, 63
11971        vpsrlq	xmm4, xmm1, 1
11972        vpslldq	xmm2, xmm5, 8
11973        vpsrldq	xmm5, xmm5, 8
11974        vpshufd	xmm2, xmm2, 255
11975        vpor	xmm4, xmm4, xmm5
11976        vpsrad	xmm2, xmm2, 31
11977        vpand	xmm2, xmm2, xmm10
11978        vpxor	xmm2, xmm2, xmm4
11979        vpsllq	xmm5, xmm2, 63
11980        vpsrlq	xmm4, xmm2, 1
11981        vpslldq	xmm3, xmm5, 8
11982        vpsrldq	xmm5, xmm5, 8
11983        vpshufd	xmm3, xmm3, 255
11984        vpor	xmm4, xmm4, xmm5
11985        vpsrad	xmm3, xmm3, 31
11986        vpand	xmm3, xmm3, xmm10
11987        vpxor	xmm3, xmm3, xmm4
11988        vpshufb	xmm2, xmm2, xmm9
11989        vpshufb	xmm3, xmm3, xmm9
11990        vpshufb	xmm0, xmm0, xmm9
11991        vpshufb	xmm1, xmm1, xmm9
11992        vpxor	xmm8, xmm3, xmm2
11993        vmovdqu	OWORD PTR [rdx+16], xmm3
11994        vmovdqu	OWORD PTR [rdx+32], xmm2
11995        vmovdqu	OWORD PTR [rdx+48], xmm8
11996        vmovdqu	OWORD PTR [rdx+64], xmm1
11997        vpxor	xmm4, xmm3, xmm1
11998        vpxor	xmm5, xmm2, xmm1
11999        vpxor	xmm6, xmm8, xmm1
12000        vmovdqu	OWORD PTR [rdx+80], xmm4
12001        vmovdqu	OWORD PTR [rdx+96], xmm5
12002        vmovdqu	OWORD PTR [rdx+112], xmm6
12003        vmovdqu	OWORD PTR [rdx+128], xmm0
12004        vpxor	xmm1, xmm1, xmm0
12005        vpxor	xmm4, xmm3, xmm0
12006        vpxor	xmm6, xmm2, xmm0
12007        vmovdqu	OWORD PTR [rdx+144], xmm4
12008        vmovdqu	OWORD PTR [rdx+160], xmm6
12009        vpxor	xmm6, xmm3, xmm6
12010        vmovdqu	OWORD PTR [rdx+176], xmm6
12011        vmovdqu	OWORD PTR [rdx+192], xmm1
12012        vpxor	xmm4, xmm3, xmm1
12013        vpxor	xmm5, xmm2, xmm1
12014        vpxor	xmm6, xmm8, xmm1
12015        vmovdqu	OWORD PTR [rdx+208], xmm4
12016        vmovdqu	OWORD PTR [rdx+224], xmm5
12017        vmovdqu	OWORD PTR [rdx+240], xmm6
12018        vmovdqu	xmm0, OWORD PTR [rdx]
12019        vmovdqu	xmm1, OWORD PTR [rdx+16]
12020        vmovdqu	xmm2, OWORD PTR [rdx+32]
12021        vmovdqu	xmm3, OWORD PTR [rdx+48]
12022        vpshufb	xmm0, xmm0, xmm9
12023        vpshufb	xmm1, xmm1, xmm9
12024        vpshufb	xmm2, xmm2, xmm9
12025        vpshufb	xmm3, xmm3, xmm9
12026        vpsllq	xmm4, xmm0, 60
12027        vpsllq	xmm5, xmm1, 60
12028        vpsllq	xmm6, xmm2, 60
12029        vpsllq	xmm7, xmm3, 60
12030        vpsrlq	xmm0, xmm0, 4
12031        vpsrlq	xmm1, xmm1, 4
12032        vpsrlq	xmm2, xmm2, 4
12033        vpsrlq	xmm3, xmm3, 4
12034        vpsrldq	xmm4, xmm4, 8
12035        vpsrldq	xmm5, xmm5, 8
12036        vpsrldq	xmm6, xmm6, 8
12037        vpsrldq	xmm7, xmm7, 8
12038        vpor	xmm0, xmm0, xmm4
12039        vpor	xmm1, xmm1, xmm5
12040        vpor	xmm2, xmm2, xmm6
12041        vpor	xmm3, xmm3, xmm7
12042        vpshufb	xmm0, xmm0, xmm9
12043        vpshufb	xmm1, xmm1, xmm9
12044        vpshufb	xmm2, xmm2, xmm9
12045        vpshufb	xmm3, xmm3, xmm9
12046        vmovdqu	OWORD PTR [rdx+256], xmm0
12047        vmovdqu	OWORD PTR [rdx+272], xmm1
12048        vmovdqu	OWORD PTR [rdx+288], xmm2
12049        vmovdqu	OWORD PTR [rdx+304], xmm3
12050        vmovdqu	xmm0, OWORD PTR [rdx+64]
12051        vmovdqu	xmm1, OWORD PTR [rdx+80]
12052        vmovdqu	xmm2, OWORD PTR [rdx+96]
12053        vmovdqu	xmm3, OWORD PTR [rdx+112]
12054        vpshufb	xmm0, xmm0, xmm9
12055        vpshufb	xmm1, xmm1, xmm9
12056        vpshufb	xmm2, xmm2, xmm9
12057        vpshufb	xmm3, xmm3, xmm9
12058        vpsllq	xmm4, xmm0, 60
12059        vpsllq	xmm5, xmm1, 60
12060        vpsllq	xmm6, xmm2, 60
12061        vpsllq	xmm7, xmm3, 60
12062        vpsrlq	xmm0, xmm0, 4
12063        vpsrlq	xmm1, xmm1, 4
12064        vpsrlq	xmm2, xmm2, 4
12065        vpsrlq	xmm3, xmm3, 4
12066        vpsrldq	xmm4, xmm4, 8
12067        vpsrldq	xmm5, xmm5, 8
12068        vpsrldq	xmm6, xmm6, 8
12069        vpsrldq	xmm7, xmm7, 8
12070        vpor	xmm0, xmm0, xmm4
12071        vpor	xmm1, xmm1, xmm5
12072        vpor	xmm2, xmm2, xmm6
12073        vpor	xmm3, xmm3, xmm7
12074        vpshufb	xmm0, xmm0, xmm9
12075        vpshufb	xmm1, xmm1, xmm9
12076        vpshufb	xmm2, xmm2, xmm9
12077        vpshufb	xmm3, xmm3, xmm9
12078        vmovdqu	OWORD PTR [rdx+320], xmm0
12079        vmovdqu	OWORD PTR [rdx+336], xmm1
12080        vmovdqu	OWORD PTR [rdx+352], xmm2
12081        vmovdqu	OWORD PTR [rdx+368], xmm3
12082        vmovdqu	xmm0, OWORD PTR [rdx+128]
12083        vmovdqu	xmm1, OWORD PTR [rdx+144]
12084        vmovdqu	xmm2, OWORD PTR [rdx+160]
12085        vmovdqu	xmm3, OWORD PTR [rdx+176]
12086        vpshufb	xmm0, xmm0, xmm9
12087        vpshufb	xmm1, xmm1, xmm9
12088        vpshufb	xmm2, xmm2, xmm9
12089        vpshufb	xmm3, xmm3, xmm9
12090        vpsllq	xmm4, xmm0, 60
12091        vpsllq	xmm5, xmm1, 60
12092        vpsllq	xmm6, xmm2, 60
12093        vpsllq	xmm7, xmm3, 60
12094        vpsrlq	xmm0, xmm0, 4
12095        vpsrlq	xmm1, xmm1, 4
12096        vpsrlq	xmm2, xmm2, 4
12097        vpsrlq	xmm3, xmm3, 4
12098        vpsrldq	xmm4, xmm4, 8
12099        vpsrldq	xmm5, xmm5, 8
12100        vpsrldq	xmm6, xmm6, 8
12101        vpsrldq	xmm7, xmm7, 8
12102        vpor	xmm0, xmm0, xmm4
12103        vpor	xmm1, xmm1, xmm5
12104        vpor	xmm2, xmm2, xmm6
12105        vpor	xmm3, xmm3, xmm7
12106        vpshufb	xmm0, xmm0, xmm9
12107        vpshufb	xmm1, xmm1, xmm9
12108        vpshufb	xmm2, xmm2, xmm9
12109        vpshufb	xmm3, xmm3, xmm9
12110        vmovdqu	OWORD PTR [rdx+384], xmm0
12111        vmovdqu	OWORD PTR [rdx+400], xmm1
12112        vmovdqu	OWORD PTR [rdx+416], xmm2
12113        vmovdqu	OWORD PTR [rdx+432], xmm3
12114        vmovdqu	xmm0, OWORD PTR [rdx+192]
12115        vmovdqu	xmm1, OWORD PTR [rdx+208]
12116        vmovdqu	xmm2, OWORD PTR [rdx+224]
12117        vmovdqu	xmm3, OWORD PTR [rdx+240]
12118        vpshufb	xmm0, xmm0, xmm9
12119        vpshufb	xmm1, xmm1, xmm9
12120        vpshufb	xmm2, xmm2, xmm9
12121        vpshufb	xmm3, xmm3, xmm9
12122        vpsllq	xmm4, xmm0, 60
12123        vpsllq	xmm5, xmm1, 60
12124        vpsllq	xmm6, xmm2, 60
12125        vpsllq	xmm7, xmm3, 60
12126        vpsrlq	xmm0, xmm0, 4
12127        vpsrlq	xmm1, xmm1, 4
12128        vpsrlq	xmm2, xmm2, 4
12129        vpsrlq	xmm3, xmm3, 4
12130        vpsrldq	xmm4, xmm4, 8
12131        vpsrldq	xmm5, xmm5, 8
12132        vpsrldq	xmm6, xmm6, 8
12133        vpsrldq	xmm7, xmm7, 8
12134        vpor	xmm0, xmm0, xmm4
12135        vpor	xmm1, xmm1, xmm5
12136        vpor	xmm2, xmm2, xmm6
12137        vpor	xmm3, xmm3, xmm7
12138        vpshufb	xmm0, xmm0, xmm9
12139        vpshufb	xmm1, xmm1, xmm9
12140        vpshufb	xmm2, xmm2, xmm9
12141        vpshufb	xmm3, xmm3, xmm9
12142        vmovdqu	OWORD PTR [rdx+448], xmm0
12143        vmovdqu	OWORD PTR [rdx+464], xmm1
12144        vmovdqu	OWORD PTR [rdx+480], xmm2
12145        vmovdqu	OWORD PTR [rdx+496], xmm3
12146        vmovdqu	xmm6, OWORD PTR [rsp]
12147        vmovdqu	xmm7, OWORD PTR [rsp+16]
12148        vmovdqu	xmm8, OWORD PTR [rsp+32]
12149        vmovdqu	xmm9, OWORD PTR [rsp+48]
12150        vmovdqu	xmm10, OWORD PTR [rsp+64]
12151        add	rsp, 80
12152        ret
12153GCM_generate_m0_avx2 ENDP
12154_TEXT ENDS
12155_DATA SEGMENT
12156ALIGN 16
12157L_avx2_aes_gcm_one QWORD \
12158     0000000000000000h,  0000000000000001h
12159ptr_L_avx2_aes_gcm_one QWORD L_avx2_aes_gcm_one
12160_DATA ENDS
12161_DATA SEGMENT
12162ALIGN 16
12163L_avx2_aes_gcm_two QWORD \
12164     0000000000000000h,  0000000000000002h
12165ptr_L_avx2_aes_gcm_two QWORD L_avx2_aes_gcm_two
12166_DATA ENDS
12167_DATA SEGMENT
12168ALIGN 16
12169L_avx2_aes_gcm_three QWORD \
12170     0000000000000000h,  0000000000000003h
12171ptr_L_avx2_aes_gcm_three QWORD L_avx2_aes_gcm_three
12172_DATA ENDS
12173_DATA SEGMENT
12174ALIGN 16
12175L_avx2_aes_gcm_four QWORD \
12176     0000000000000000h,  0000000000000004h
12177ptr_L_avx2_aes_gcm_four QWORD L_avx2_aes_gcm_four
12178_DATA ENDS
12179_DATA SEGMENT
12180ALIGN 16
12181L_avx2_aes_gcm_five QWORD \
12182     0000000000000000h,  0000000000000005h
12183ptr_L_avx2_aes_gcm_five QWORD L_avx2_aes_gcm_five
12184_DATA ENDS
12185_DATA SEGMENT
12186ALIGN 16
12187L_avx2_aes_gcm_six QWORD \
12188     0000000000000000h,  0000000000000006h
12189ptr_L_avx2_aes_gcm_six QWORD L_avx2_aes_gcm_six
12190_DATA ENDS
12191_DATA SEGMENT
12192ALIGN 16
12193L_avx2_aes_gcm_seven QWORD \
12194     0000000000000000h,  0000000000000007h
12195ptr_L_avx2_aes_gcm_seven QWORD L_avx2_aes_gcm_seven
12196_DATA ENDS
12197_DATA SEGMENT
12198ALIGN 16
12199L_avx2_aes_gcm_eight QWORD \
12200     0000000000000000h,  0000000000000008h
12201ptr_L_avx2_aes_gcm_eight QWORD L_avx2_aes_gcm_eight
12202_DATA ENDS
12203_DATA SEGMENT
12204ALIGN 16
12205L_avx2_aes_gcm_bswap_one QWORD \
12206     0000000000000000h,  0100000000000000h
12207ptr_L_avx2_aes_gcm_bswap_one QWORD L_avx2_aes_gcm_bswap_one
12208_DATA ENDS
12209_DATA SEGMENT
12210ALIGN 16
12211L_avx2_aes_gcm_bswap_epi64 QWORD \
12212     0001020304050607h,  08090a0b0c0d0e0fh
12213ptr_L_avx2_aes_gcm_bswap_epi64 QWORD L_avx2_aes_gcm_bswap_epi64
12214_DATA ENDS
12215_DATA SEGMENT
12216ALIGN 16
12217L_avx2_aes_gcm_bswap_mask QWORD \
12218     08090a0b0c0d0e0fh,  0001020304050607h
12219ptr_L_avx2_aes_gcm_bswap_mask QWORD L_avx2_aes_gcm_bswap_mask
12220_DATA ENDS
12221_DATA SEGMENT
12222ALIGN 16
12223L_avx2_aes_gcm_mod2_128 QWORD \
12224     0000000000000001h, 0c200000000000000h
12225ptr_L_avx2_aes_gcm_mod2_128 QWORD L_avx2_aes_gcm_mod2_128
12226_DATA ENDS
12227_TEXT SEGMENT READONLY PARA
12228AES_GCM_encrypt_avx2 PROC
12229        push	r13
12230        push	rdi
12231        push	r12
12232        push	r15
12233        push	rbx
12234        push	r14
12235        push	rsi
12236        mov	rdi, rcx
12237        mov	r12, r8
12238        mov	rax, r9
12239        mov	r15, QWORD PTR [rsp+96]
12240        mov	r8, rdx
12241        mov	r10d, DWORD PTR [rsp+104]
12242        mov	r11d, DWORD PTR [rsp+112]
12243        mov	ebx, DWORD PTR [rsp+120]
12244        mov	r14d, DWORD PTR [rsp+128]
12245        mov	rsi, QWORD PTR [rsp+136]
12246        mov	r9d, DWORD PTR [rsp+144]
12247        sub	rsp, 320
12248        vmovdqu	OWORD PTR [rsp+160], xmm6
12249        vmovdqu	OWORD PTR [rsp+176], xmm7
12250        vmovdqu	OWORD PTR [rsp+192], xmm8
12251        vmovdqu	OWORD PTR [rsp+208], xmm9
12252        vmovdqu	OWORD PTR [rsp+224], xmm10
12253        vmovdqu	OWORD PTR [rsp+240], xmm11
12254        vmovdqu	OWORD PTR [rsp+256], xmm12
12255        vmovdqu	OWORD PTR [rsp+272], xmm13
12256        vmovdqu	OWORD PTR [rsp+288], xmm14
12257        vmovdqu	OWORD PTR [rsp+304], xmm15
12258        vpxor	xmm4, xmm4, xmm4
12259        vpxor	xmm6, xmm6, xmm6
12260        mov	edx, ebx
12261        cmp	edx, 12
12262        je	L_AES_GCM_encrypt_avx2_iv_12
12263        ; Calculate values when IV is not 12 bytes
12264        ; H = Encrypt X(=0)
12265        vmovdqu	xmm5, OWORD PTR [rsi]
12266        vaesenc	xmm5, xmm5, [rsi+16]
12267        vaesenc	xmm5, xmm5, [rsi+32]
12268        vaesenc	xmm5, xmm5, [rsi+48]
12269        vaesenc	xmm5, xmm5, [rsi+64]
12270        vaesenc	xmm5, xmm5, [rsi+80]
12271        vaesenc	xmm5, xmm5, [rsi+96]
12272        vaesenc	xmm5, xmm5, [rsi+112]
12273        vaesenc	xmm5, xmm5, [rsi+128]
12274        vaesenc	xmm5, xmm5, [rsi+144]
12275        cmp	r9d, 11
12276        vmovdqu	xmm0, OWORD PTR [rsi+160]
12277        jl	L_AES_GCM_encrypt_avx2_calc_iv_1_aesenc_avx_last
12278        vaesenc	xmm5, xmm5, xmm0
12279        vaesenc	xmm5, xmm5, [rsi+176]
12280        cmp	r9d, 13
12281        vmovdqu	xmm0, OWORD PTR [rsi+192]
12282        jl	L_AES_GCM_encrypt_avx2_calc_iv_1_aesenc_avx_last
12283        vaesenc	xmm5, xmm5, xmm0
12284        vaesenc	xmm5, xmm5, [rsi+208]
12285        vmovdqu	xmm0, OWORD PTR [rsi+224]
12286L_AES_GCM_encrypt_avx2_calc_iv_1_aesenc_avx_last:
12287        vaesenclast	xmm5, xmm5, xmm0
12288        vpshufb	xmm5, xmm5, OWORD PTR L_avx2_aes_gcm_bswap_mask
12289        ; Calc counter
12290        ; Initialization vector
12291        cmp	edx, 0
12292        mov	rcx, 0
12293        je	L_AES_GCM_encrypt_avx2_calc_iv_done
12294        cmp	edx, 16
12295        jl	L_AES_GCM_encrypt_avx2_calc_iv_lt16
12296        and	edx, 4294967280
12297L_AES_GCM_encrypt_avx2_calc_iv_16_loop:
12298        vmovdqu	xmm0, OWORD PTR [rax+rcx]
12299        vpshufb	xmm0, xmm0, OWORD PTR L_avx2_aes_gcm_bswap_mask
12300        vpxor	xmm4, xmm4, xmm0
12301        ; ghash_gfmul_avx
12302        vpclmulqdq	xmm2, xmm5, xmm4, 16
12303        vpclmulqdq	xmm1, xmm5, xmm4, 1
12304        vpclmulqdq	xmm0, xmm5, xmm4, 0
12305        vpclmulqdq	xmm3, xmm5, xmm4, 17
12306        vpxor	xmm2, xmm2, xmm1
12307        vpslldq	xmm1, xmm2, 8
12308        vpsrldq	xmm2, xmm2, 8
12309        vpxor	xmm7, xmm0, xmm1
12310        vpxor	xmm4, xmm3, xmm2
12311        ; ghash_mid
12312        vpsrld	xmm0, xmm7, 31
12313        vpsrld	xmm1, xmm4, 31
12314        vpslld	xmm7, xmm7, 1
12315        vpslld	xmm4, xmm4, 1
12316        vpsrldq	xmm2, xmm0, 12
12317        vpslldq	xmm0, xmm0, 4
12318        vpslldq	xmm1, xmm1, 4
12319        vpor	xmm4, xmm4, xmm2
12320        vpor	xmm7, xmm7, xmm0
12321        vpor	xmm4, xmm4, xmm1
12322        ; ghash_red
12323        vmovdqu	xmm2, OWORD PTR L_avx2_aes_gcm_mod2_128
12324        vpclmulqdq	xmm0, xmm7, xmm2, 16
12325        vpshufd	xmm1, xmm7, 78
12326        vpxor	xmm1, xmm1, xmm0
12327        vpclmulqdq	xmm0, xmm1, xmm2, 16
12328        vpshufd	xmm1, xmm1, 78
12329        vpxor	xmm1, xmm1, xmm0
12330        vpxor	xmm4, xmm4, xmm1
12331        add	ecx, 16
12332        cmp	ecx, edx
12333        jl	L_AES_GCM_encrypt_avx2_calc_iv_16_loop
12334        mov	edx, ebx
12335        cmp	ecx, edx
12336        je	L_AES_GCM_encrypt_avx2_calc_iv_done
12337L_AES_GCM_encrypt_avx2_calc_iv_lt16:
12338        vpxor	xmm0, xmm0, xmm0
12339        xor	ebx, ebx
12340        vmovdqu	OWORD PTR [rsp], xmm0
12341L_AES_GCM_encrypt_avx2_calc_iv_loop:
12342        movzx	r13d, BYTE PTR [rax+rcx]
12343        mov	BYTE PTR [rsp+rbx], r13b
12344        inc	ecx
12345        inc	ebx
12346        cmp	ecx, edx
12347        jl	L_AES_GCM_encrypt_avx2_calc_iv_loop
12348        vmovdqu	xmm0, OWORD PTR [rsp]
12349        vpshufb	xmm0, xmm0, OWORD PTR L_avx2_aes_gcm_bswap_mask
12350        vpxor	xmm4, xmm4, xmm0
12351        ; ghash_gfmul_avx
12352        vpclmulqdq	xmm2, xmm5, xmm4, 16
12353        vpclmulqdq	xmm1, xmm5, xmm4, 1
12354        vpclmulqdq	xmm0, xmm5, xmm4, 0
12355        vpclmulqdq	xmm3, xmm5, xmm4, 17
12356        vpxor	xmm2, xmm2, xmm1
12357        vpslldq	xmm1, xmm2, 8
12358        vpsrldq	xmm2, xmm2, 8
12359        vpxor	xmm7, xmm0, xmm1
12360        vpxor	xmm4, xmm3, xmm2
12361        ; ghash_mid
12362        vpsrld	xmm0, xmm7, 31
12363        vpsrld	xmm1, xmm4, 31
12364        vpslld	xmm7, xmm7, 1
12365        vpslld	xmm4, xmm4, 1
12366        vpsrldq	xmm2, xmm0, 12
12367        vpslldq	xmm0, xmm0, 4
12368        vpslldq	xmm1, xmm1, 4
12369        vpor	xmm4, xmm4, xmm2
12370        vpor	xmm7, xmm7, xmm0
12371        vpor	xmm4, xmm4, xmm1
12372        ; ghash_red
12373        vmovdqu	xmm2, OWORD PTR L_avx2_aes_gcm_mod2_128
12374        vpclmulqdq	xmm0, xmm7, xmm2, 16
12375        vpshufd	xmm1, xmm7, 78
12376        vpxor	xmm1, xmm1, xmm0
12377        vpclmulqdq	xmm0, xmm1, xmm2, 16
12378        vpshufd	xmm1, xmm1, 78
12379        vpxor	xmm1, xmm1, xmm0
12380        vpxor	xmm4, xmm4, xmm1
12381L_AES_GCM_encrypt_avx2_calc_iv_done:
12382        ; T = Encrypt counter
12383        vpxor	xmm0, xmm0, xmm0
12384        shl	edx, 3
12385        vmovq	xmm0, rdx
12386        vpxor	xmm4, xmm4, xmm0
12387        ; ghash_gfmul_avx
12388        vpclmulqdq	xmm2, xmm5, xmm4, 16
12389        vpclmulqdq	xmm1, xmm5, xmm4, 1
12390        vpclmulqdq	xmm0, xmm5, xmm4, 0
12391        vpclmulqdq	xmm3, xmm5, xmm4, 17
12392        vpxor	xmm2, xmm2, xmm1
12393        vpslldq	xmm1, xmm2, 8
12394        vpsrldq	xmm2, xmm2, 8
12395        vpxor	xmm7, xmm0, xmm1
12396        vpxor	xmm4, xmm3, xmm2
12397        ; ghash_mid
12398        vpsrld	xmm0, xmm7, 31
12399        vpsrld	xmm1, xmm4, 31
12400        vpslld	xmm7, xmm7, 1
12401        vpslld	xmm4, xmm4, 1
12402        vpsrldq	xmm2, xmm0, 12
12403        vpslldq	xmm0, xmm0, 4
12404        vpslldq	xmm1, xmm1, 4
12405        vpor	xmm4, xmm4, xmm2
12406        vpor	xmm7, xmm7, xmm0
12407        vpor	xmm4, xmm4, xmm1
12408        ; ghash_red
12409        vmovdqu	xmm2, OWORD PTR L_avx2_aes_gcm_mod2_128
12410        vpclmulqdq	xmm0, xmm7, xmm2, 16
12411        vpshufd	xmm1, xmm7, 78
12412        vpxor	xmm1, xmm1, xmm0
12413        vpclmulqdq	xmm0, xmm1, xmm2, 16
12414        vpshufd	xmm1, xmm1, 78
12415        vpxor	xmm1, xmm1, xmm0
12416        vpxor	xmm4, xmm4, xmm1
12417        vpshufb	xmm4, xmm4, OWORD PTR L_avx2_aes_gcm_bswap_mask
12418        ;   Encrypt counter
12419        vmovdqu	xmm15, OWORD PTR [rsi]
12420        vpxor	xmm15, xmm15, xmm4
12421        vaesenc	xmm15, xmm15, [rsi+16]
12422        vaesenc	xmm15, xmm15, [rsi+32]
12423        vaesenc	xmm15, xmm15, [rsi+48]
12424        vaesenc	xmm15, xmm15, [rsi+64]
12425        vaesenc	xmm15, xmm15, [rsi+80]
12426        vaesenc	xmm15, xmm15, [rsi+96]
12427        vaesenc	xmm15, xmm15, [rsi+112]
12428        vaesenc	xmm15, xmm15, [rsi+128]
12429        vaesenc	xmm15, xmm15, [rsi+144]
12430        cmp	r9d, 11
12431        vmovdqu	xmm0, OWORD PTR [rsi+160]
12432        jl	L_AES_GCM_encrypt_avx2_calc_iv_2_aesenc_avx_last
12433        vaesenc	xmm15, xmm15, xmm0
12434        vaesenc	xmm15, xmm15, [rsi+176]
12435        cmp	r9d, 13
12436        vmovdqu	xmm0, OWORD PTR [rsi+192]
12437        jl	L_AES_GCM_encrypt_avx2_calc_iv_2_aesenc_avx_last
12438        vaesenc	xmm15, xmm15, xmm0
12439        vaesenc	xmm15, xmm15, [rsi+208]
12440        vmovdqu	xmm0, OWORD PTR [rsi+224]
12441L_AES_GCM_encrypt_avx2_calc_iv_2_aesenc_avx_last:
12442        vaesenclast	xmm15, xmm15, xmm0
12443        jmp	L_AES_GCM_encrypt_avx2_iv_done
12444L_AES_GCM_encrypt_avx2_iv_12:
12445        ; # Calculate values when IV is 12 bytes
12446        ; Set counter based on IV
12447        vmovdqu	xmm4, OWORD PTR L_avx2_aes_gcm_bswap_one
12448        vmovdqu	xmm5, OWORD PTR [rsi]
12449        vpblendd	xmm4, xmm4, [rax], 7
12450        ; H = Encrypt X(=0) and T = Encrypt counter
12451        vmovdqu	xmm7, OWORD PTR [rsi+16]
12452        vpxor	xmm15, xmm4, xmm5
12453        vaesenc	xmm5, xmm5, xmm7
12454        vaesenc	xmm15, xmm15, xmm7
12455        vmovdqu	xmm0, OWORD PTR [rsi+32]
12456        vaesenc	xmm5, xmm5, xmm0
12457        vaesenc	xmm15, xmm15, xmm0
12458        vmovdqu	xmm0, OWORD PTR [rsi+48]
12459        vaesenc	xmm5, xmm5, xmm0
12460        vaesenc	xmm15, xmm15, xmm0
12461        vmovdqu	xmm0, OWORD PTR [rsi+64]
12462        vaesenc	xmm5, xmm5, xmm0
12463        vaesenc	xmm15, xmm15, xmm0
12464        vmovdqu	xmm0, OWORD PTR [rsi+80]
12465        vaesenc	xmm5, xmm5, xmm0
12466        vaesenc	xmm15, xmm15, xmm0
12467        vmovdqu	xmm0, OWORD PTR [rsi+96]
12468        vaesenc	xmm5, xmm5, xmm0
12469        vaesenc	xmm15, xmm15, xmm0
12470        vmovdqu	xmm0, OWORD PTR [rsi+112]
12471        vaesenc	xmm5, xmm5, xmm0
12472        vaesenc	xmm15, xmm15, xmm0
12473        vmovdqu	xmm0, OWORD PTR [rsi+128]
12474        vaesenc	xmm5, xmm5, xmm0
12475        vaesenc	xmm15, xmm15, xmm0
12476        vmovdqu	xmm0, OWORD PTR [rsi+144]
12477        vaesenc	xmm5, xmm5, xmm0
12478        vaesenc	xmm15, xmm15, xmm0
12479        cmp	r9d, 11
12480        vmovdqu	xmm0, OWORD PTR [rsi+160]
12481        jl	L_AES_GCM_encrypt_avx2_calc_iv_12_last
12482        vaesenc	xmm5, xmm5, xmm0
12483        vaesenc	xmm15, xmm15, xmm0
12484        vmovdqu	xmm0, OWORD PTR [rsi+176]
12485        vaesenc	xmm5, xmm5, xmm0
12486        vaesenc	xmm15, xmm15, xmm0
12487        cmp	r9d, 13
12488        vmovdqu	xmm0, OWORD PTR [rsi+192]
12489        jl	L_AES_GCM_encrypt_avx2_calc_iv_12_last
12490        vaesenc	xmm5, xmm5, xmm0
12491        vaesenc	xmm15, xmm15, xmm0
12492        vmovdqu	xmm0, OWORD PTR [rsi+208]
12493        vaesenc	xmm5, xmm5, xmm0
12494        vaesenc	xmm15, xmm15, xmm0
12495        vmovdqu	xmm0, OWORD PTR [rsi+224]
12496L_AES_GCM_encrypt_avx2_calc_iv_12_last:
12497        vaesenclast	xmm5, xmm5, xmm0
12498        vaesenclast	xmm15, xmm15, xmm0
12499        vpshufb	xmm5, xmm5, OWORD PTR L_avx2_aes_gcm_bswap_mask
12500L_AES_GCM_encrypt_avx2_iv_done:
12501        ; Additional authentication data
12502        mov	edx, r11d
12503        cmp	edx, 0
12504        je	L_AES_GCM_encrypt_avx2_calc_aad_done
12505        xor	ecx, ecx
12506        cmp	edx, 16
12507        jl	L_AES_GCM_encrypt_avx2_calc_aad_lt16
12508        and	edx, 4294967280
12509L_AES_GCM_encrypt_avx2_calc_aad_16_loop:
12510        vmovdqu	xmm0, OWORD PTR [r12+rcx]
12511        vpshufb	xmm0, xmm0, OWORD PTR L_avx2_aes_gcm_bswap_mask
12512        vpxor	xmm6, xmm6, xmm0
12513        ; ghash_gfmul_avx
12514        vpclmulqdq	xmm2, xmm5, xmm6, 16
12515        vpclmulqdq	xmm1, xmm5, xmm6, 1
12516        vpclmulqdq	xmm0, xmm5, xmm6, 0
12517        vpclmulqdq	xmm3, xmm5, xmm6, 17
12518        vpxor	xmm2, xmm2, xmm1
12519        vpslldq	xmm1, xmm2, 8
12520        vpsrldq	xmm2, xmm2, 8
12521        vpxor	xmm7, xmm0, xmm1
12522        vpxor	xmm6, xmm3, xmm2
12523        ; ghash_mid
12524        vpsrld	xmm0, xmm7, 31
12525        vpsrld	xmm1, xmm6, 31
12526        vpslld	xmm7, xmm7, 1
12527        vpslld	xmm6, xmm6, 1
12528        vpsrldq	xmm2, xmm0, 12
12529        vpslldq	xmm0, xmm0, 4
12530        vpslldq	xmm1, xmm1, 4
12531        vpor	xmm6, xmm6, xmm2
12532        vpor	xmm7, xmm7, xmm0
12533        vpor	xmm6, xmm6, xmm1
12534        ; ghash_red
12535        vmovdqu	xmm2, OWORD PTR L_avx2_aes_gcm_mod2_128
12536        vpclmulqdq	xmm0, xmm7, xmm2, 16
12537        vpshufd	xmm1, xmm7, 78
12538        vpxor	xmm1, xmm1, xmm0
12539        vpclmulqdq	xmm0, xmm1, xmm2, 16
12540        vpshufd	xmm1, xmm1, 78
12541        vpxor	xmm1, xmm1, xmm0
12542        vpxor	xmm6, xmm6, xmm1
12543        add	ecx, 16
12544        cmp	ecx, edx
12545        jl	L_AES_GCM_encrypt_avx2_calc_aad_16_loop
12546        mov	edx, r11d
12547        cmp	ecx, edx
12548        je	L_AES_GCM_encrypt_avx2_calc_aad_done
12549L_AES_GCM_encrypt_avx2_calc_aad_lt16:
12550        vpxor	xmm0, xmm0, xmm0
12551        xor	ebx, ebx
12552        vmovdqu	OWORD PTR [rsp], xmm0
12553L_AES_GCM_encrypt_avx2_calc_aad_loop:
12554        movzx	r13d, BYTE PTR [r12+rcx]
12555        mov	BYTE PTR [rsp+rbx], r13b
12556        inc	ecx
12557        inc	ebx
12558        cmp	ecx, edx
12559        jl	L_AES_GCM_encrypt_avx2_calc_aad_loop
12560        vmovdqu	xmm0, OWORD PTR [rsp]
12561        vpshufb	xmm0, xmm0, OWORD PTR L_avx2_aes_gcm_bswap_mask
12562        vpxor	xmm6, xmm6, xmm0
12563        ; ghash_gfmul_avx
12564        vpclmulqdq	xmm2, xmm5, xmm6, 16
12565        vpclmulqdq	xmm1, xmm5, xmm6, 1
12566        vpclmulqdq	xmm0, xmm5, xmm6, 0
12567        vpclmulqdq	xmm3, xmm5, xmm6, 17
12568        vpxor	xmm2, xmm2, xmm1
12569        vpslldq	xmm1, xmm2, 8
12570        vpsrldq	xmm2, xmm2, 8
12571        vpxor	xmm7, xmm0, xmm1
12572        vpxor	xmm6, xmm3, xmm2
12573        ; ghash_mid
12574        vpsrld	xmm0, xmm7, 31
12575        vpsrld	xmm1, xmm6, 31
12576        vpslld	xmm7, xmm7, 1
12577        vpslld	xmm6, xmm6, 1
12578        vpsrldq	xmm2, xmm0, 12
12579        vpslldq	xmm0, xmm0, 4
12580        vpslldq	xmm1, xmm1, 4
12581        vpor	xmm6, xmm6, xmm2
12582        vpor	xmm7, xmm7, xmm0
12583        vpor	xmm6, xmm6, xmm1
12584        ; ghash_red
12585        vmovdqu	xmm2, OWORD PTR L_avx2_aes_gcm_mod2_128
12586        vpclmulqdq	xmm0, xmm7, xmm2, 16
12587        vpshufd	xmm1, xmm7, 78
12588        vpxor	xmm1, xmm1, xmm0
12589        vpclmulqdq	xmm0, xmm1, xmm2, 16
12590        vpshufd	xmm1, xmm1, 78
12591        vpxor	xmm1, xmm1, xmm0
12592        vpxor	xmm6, xmm6, xmm1
12593L_AES_GCM_encrypt_avx2_calc_aad_done:
12594        ; Calculate counter and H
12595        vpsrlq	xmm1, xmm5, 63
12596        vpsllq	xmm0, xmm5, 1
12597        vpslldq	xmm1, xmm1, 8
12598        vpor	xmm0, xmm0, xmm1
12599        vpshufd	xmm5, xmm5, 255
12600        vpsrad	xmm5, xmm5, 31
12601        vpshufb	xmm4, xmm4, OWORD PTR L_avx2_aes_gcm_bswap_epi64
12602        vpand	xmm5, xmm5, OWORD PTR L_avx2_aes_gcm_mod2_128
12603        vpaddd	xmm4, xmm4, OWORD PTR L_avx2_aes_gcm_one
12604        vpxor	xmm5, xmm5, xmm0
12605        xor	ebx, ebx
12606        cmp	r10d, 128
12607        mov	r13d, r10d
12608        jl	L_AES_GCM_encrypt_avx2_done_128
12609        and	r13d, 4294967168
12610        vmovdqu	OWORD PTR [rsp+128], xmm4
12611        vmovdqu	OWORD PTR [rsp+144], xmm15
12612        vmovdqu	xmm3, OWORD PTR L_avx2_aes_gcm_mod2_128
12613        ; H ^ 1 and H ^ 2
12614        vpclmulqdq	xmm9, xmm5, xmm5, 0
12615        vpclmulqdq	xmm10, xmm5, xmm5, 17
12616        vpclmulqdq	xmm8, xmm9, xmm3, 16
12617        vpshufd	xmm9, xmm9, 78
12618        vpxor	xmm9, xmm9, xmm8
12619        vpclmulqdq	xmm8, xmm9, xmm3, 16
12620        vpshufd	xmm9, xmm9, 78
12621        vpxor	xmm9, xmm9, xmm8
12622        vpxor	xmm0, xmm10, xmm9
12623        vmovdqu	OWORD PTR [rsp], xmm5
12624        vmovdqu	OWORD PTR [rsp+16], xmm0
12625        ; H ^ 3 and H ^ 4
12626        vpclmulqdq	xmm11, xmm0, xmm5, 16
12627        vpclmulqdq	xmm10, xmm0, xmm5, 1
12628        vpclmulqdq	xmm9, xmm0, xmm5, 0
12629        vpclmulqdq	xmm12, xmm0, xmm5, 17
12630        vpclmulqdq	xmm13, xmm0, xmm0, 0
12631        vpclmulqdq	xmm14, xmm0, xmm0, 17
12632        vpxor	xmm11, xmm11, xmm10
12633        vpslldq	xmm10, xmm11, 8
12634        vpsrldq	xmm11, xmm11, 8
12635        vpxor	xmm10, xmm10, xmm9
12636        vpclmulqdq	xmm8, xmm13, xmm3, 16
12637        vpclmulqdq	xmm9, xmm10, xmm3, 16
12638        vpshufd	xmm10, xmm10, 78
12639        vpshufd	xmm13, xmm13, 78
12640        vpxor	xmm10, xmm10, xmm9
12641        vpxor	xmm13, xmm13, xmm8
12642        vpclmulqdq	xmm9, xmm10, xmm3, 16
12643        vpclmulqdq	xmm8, xmm13, xmm3, 16
12644        vpshufd	xmm10, xmm10, 78
12645        vpshufd	xmm13, xmm13, 78
12646        vpxor	xmm12, xmm12, xmm11
12647        vpxor	xmm13, xmm13, xmm8
12648        vpxor	xmm10, xmm10, xmm12
12649        vpxor	xmm2, xmm13, xmm14
12650        vpxor	xmm1, xmm10, xmm9
12651        vmovdqu	OWORD PTR [rsp+32], xmm1
12652        vmovdqu	OWORD PTR [rsp+48], xmm2
12653        ; H ^ 5 and H ^ 6
12654        vpclmulqdq	xmm11, xmm1, xmm0, 16
12655        vpclmulqdq	xmm10, xmm1, xmm0, 1
12656        vpclmulqdq	xmm9, xmm1, xmm0, 0
12657        vpclmulqdq	xmm12, xmm1, xmm0, 17
12658        vpclmulqdq	xmm13, xmm1, xmm1, 0
12659        vpclmulqdq	xmm14, xmm1, xmm1, 17
12660        vpxor	xmm11, xmm11, xmm10
12661        vpslldq	xmm10, xmm11, 8
12662        vpsrldq	xmm11, xmm11, 8
12663        vpxor	xmm10, xmm10, xmm9
12664        vpclmulqdq	xmm8, xmm13, xmm3, 16
12665        vpclmulqdq	xmm9, xmm10, xmm3, 16
12666        vpshufd	xmm10, xmm10, 78
12667        vpshufd	xmm13, xmm13, 78
12668        vpxor	xmm10, xmm10, xmm9
12669        vpxor	xmm13, xmm13, xmm8
12670        vpclmulqdq	xmm9, xmm10, xmm3, 16
12671        vpclmulqdq	xmm8, xmm13, xmm3, 16
12672        vpshufd	xmm10, xmm10, 78
12673        vpshufd	xmm13, xmm13, 78
12674        vpxor	xmm12, xmm12, xmm11
12675        vpxor	xmm13, xmm13, xmm8
12676        vpxor	xmm10, xmm10, xmm12
12677        vpxor	xmm0, xmm13, xmm14
12678        vpxor	xmm7, xmm10, xmm9
12679        vmovdqu	OWORD PTR [rsp+64], xmm7
12680        vmovdqu	OWORD PTR [rsp+80], xmm0
12681        ; H ^ 7 and H ^ 8
12682        vpclmulqdq	xmm11, xmm2, xmm1, 16
12683        vpclmulqdq	xmm10, xmm2, xmm1, 1
12684        vpclmulqdq	xmm9, xmm2, xmm1, 0
12685        vpclmulqdq	xmm12, xmm2, xmm1, 17
12686        vpclmulqdq	xmm13, xmm2, xmm2, 0
12687        vpclmulqdq	xmm14, xmm2, xmm2, 17
12688        vpxor	xmm11, xmm11, xmm10
12689        vpslldq	xmm10, xmm11, 8
12690        vpsrldq	xmm11, xmm11, 8
12691        vpxor	xmm10, xmm10, xmm9
12692        vpclmulqdq	xmm8, xmm13, xmm3, 16
12693        vpclmulqdq	xmm9, xmm10, xmm3, 16
12694        vpshufd	xmm10, xmm10, 78
12695        vpshufd	xmm13, xmm13, 78
12696        vpxor	xmm10, xmm10, xmm9
12697        vpxor	xmm13, xmm13, xmm8
12698        vpclmulqdq	xmm9, xmm10, xmm3, 16
12699        vpclmulqdq	xmm8, xmm13, xmm3, 16
12700        vpshufd	xmm10, xmm10, 78
12701        vpshufd	xmm13, xmm13, 78
12702        vpxor	xmm12, xmm12, xmm11
12703        vpxor	xmm13, xmm13, xmm8
12704        vpxor	xmm10, xmm10, xmm12
12705        vpxor	xmm0, xmm13, xmm14
12706        vpxor	xmm7, xmm10, xmm9
12707        vmovdqu	OWORD PTR [rsp+96], xmm7
12708        vmovdqu	OWORD PTR [rsp+112], xmm0
12709        ; First 128 bytes of input
12710        ; aesenc_128
12711        ; aesenc_ctr
12712        vmovdqu	xmm0, OWORD PTR [rsp+128]
12713        vmovdqu	xmm1, OWORD PTR L_avx2_aes_gcm_bswap_epi64
12714        vpaddd	xmm9, xmm0, OWORD PTR L_avx2_aes_gcm_one
12715        vpshufb	xmm8, xmm0, xmm1
12716        vpaddd	xmm10, xmm0, OWORD PTR L_avx2_aes_gcm_two
12717        vpshufb	xmm9, xmm9, xmm1
12718        vpaddd	xmm11, xmm0, OWORD PTR L_avx2_aes_gcm_three
12719        vpshufb	xmm10, xmm10, xmm1
12720        vpaddd	xmm12, xmm0, OWORD PTR L_avx2_aes_gcm_four
12721        vpshufb	xmm11, xmm11, xmm1
12722        vpaddd	xmm13, xmm0, OWORD PTR L_avx2_aes_gcm_five
12723        vpshufb	xmm12, xmm12, xmm1
12724        vpaddd	xmm14, xmm0, OWORD PTR L_avx2_aes_gcm_six
12725        vpshufb	xmm13, xmm13, xmm1
12726        vpaddd	xmm15, xmm0, OWORD PTR L_avx2_aes_gcm_seven
12727        vpshufb	xmm14, xmm14, xmm1
12728        vpaddd	xmm0, xmm0, OWORD PTR L_avx2_aes_gcm_eight
12729        vpshufb	xmm15, xmm15, xmm1
12730        ; aesenc_xor
12731        vmovdqu	xmm7, OWORD PTR [rsi]
12732        vmovdqu	OWORD PTR [rsp+128], xmm0
12733        vpxor	xmm8, xmm8, xmm7
12734        vpxor	xmm9, xmm9, xmm7
12735        vpxor	xmm10, xmm10, xmm7
12736        vpxor	xmm11, xmm11, xmm7
12737        vpxor	xmm12, xmm12, xmm7
12738        vpxor	xmm13, xmm13, xmm7
12739        vpxor	xmm14, xmm14, xmm7
12740        vpxor	xmm15, xmm15, xmm7
12741        vmovdqu	xmm7, OWORD PTR [rsi+16]
12742        vaesenc	xmm8, xmm8, xmm7
12743        vaesenc	xmm9, xmm9, xmm7
12744        vaesenc	xmm10, xmm10, xmm7
12745        vaesenc	xmm11, xmm11, xmm7
12746        vaesenc	xmm12, xmm12, xmm7
12747        vaesenc	xmm13, xmm13, xmm7
12748        vaesenc	xmm14, xmm14, xmm7
12749        vaesenc	xmm15, xmm15, xmm7
12750        vmovdqu	xmm7, OWORD PTR [rsi+32]
12751        vaesenc	xmm8, xmm8, xmm7
12752        vaesenc	xmm9, xmm9, xmm7
12753        vaesenc	xmm10, xmm10, xmm7
12754        vaesenc	xmm11, xmm11, xmm7
12755        vaesenc	xmm12, xmm12, xmm7
12756        vaesenc	xmm13, xmm13, xmm7
12757        vaesenc	xmm14, xmm14, xmm7
12758        vaesenc	xmm15, xmm15, xmm7
12759        vmovdqu	xmm7, OWORD PTR [rsi+48]
12760        vaesenc	xmm8, xmm8, xmm7
12761        vaesenc	xmm9, xmm9, xmm7
12762        vaesenc	xmm10, xmm10, xmm7
12763        vaesenc	xmm11, xmm11, xmm7
12764        vaesenc	xmm12, xmm12, xmm7
12765        vaesenc	xmm13, xmm13, xmm7
12766        vaesenc	xmm14, xmm14, xmm7
12767        vaesenc	xmm15, xmm15, xmm7
12768        vmovdqu	xmm7, OWORD PTR [rsi+64]
12769        vaesenc	xmm8, xmm8, xmm7
12770        vaesenc	xmm9, xmm9, xmm7
12771        vaesenc	xmm10, xmm10, xmm7
12772        vaesenc	xmm11, xmm11, xmm7
12773        vaesenc	xmm12, xmm12, xmm7
12774        vaesenc	xmm13, xmm13, xmm7
12775        vaesenc	xmm14, xmm14, xmm7
12776        vaesenc	xmm15, xmm15, xmm7
12777        vmovdqu	xmm7, OWORD PTR [rsi+80]
12778        vaesenc	xmm8, xmm8, xmm7
12779        vaesenc	xmm9, xmm9, xmm7
12780        vaesenc	xmm10, xmm10, xmm7
12781        vaesenc	xmm11, xmm11, xmm7
12782        vaesenc	xmm12, xmm12, xmm7
12783        vaesenc	xmm13, xmm13, xmm7
12784        vaesenc	xmm14, xmm14, xmm7
12785        vaesenc	xmm15, xmm15, xmm7
12786        vmovdqu	xmm7, OWORD PTR [rsi+96]
12787        vaesenc	xmm8, xmm8, xmm7
12788        vaesenc	xmm9, xmm9, xmm7
12789        vaesenc	xmm10, xmm10, xmm7
12790        vaesenc	xmm11, xmm11, xmm7
12791        vaesenc	xmm12, xmm12, xmm7
12792        vaesenc	xmm13, xmm13, xmm7
12793        vaesenc	xmm14, xmm14, xmm7
12794        vaesenc	xmm15, xmm15, xmm7
12795        vmovdqu	xmm7, OWORD PTR [rsi+112]
12796        vaesenc	xmm8, xmm8, xmm7
12797        vaesenc	xmm9, xmm9, xmm7
12798        vaesenc	xmm10, xmm10, xmm7
12799        vaesenc	xmm11, xmm11, xmm7
12800        vaesenc	xmm12, xmm12, xmm7
12801        vaesenc	xmm13, xmm13, xmm7
12802        vaesenc	xmm14, xmm14, xmm7
12803        vaesenc	xmm15, xmm15, xmm7
12804        vmovdqu	xmm7, OWORD PTR [rsi+128]
12805        vaesenc	xmm8, xmm8, xmm7
12806        vaesenc	xmm9, xmm9, xmm7
12807        vaesenc	xmm10, xmm10, xmm7
12808        vaesenc	xmm11, xmm11, xmm7
12809        vaesenc	xmm12, xmm12, xmm7
12810        vaesenc	xmm13, xmm13, xmm7
12811        vaesenc	xmm14, xmm14, xmm7
12812        vaesenc	xmm15, xmm15, xmm7
12813        vmovdqu	xmm7, OWORD PTR [rsi+144]
12814        vaesenc	xmm8, xmm8, xmm7
12815        vaesenc	xmm9, xmm9, xmm7
12816        vaesenc	xmm10, xmm10, xmm7
12817        vaesenc	xmm11, xmm11, xmm7
12818        vaesenc	xmm12, xmm12, xmm7
12819        vaesenc	xmm13, xmm13, xmm7
12820        vaesenc	xmm14, xmm14, xmm7
12821        vaesenc	xmm15, xmm15, xmm7
12822        cmp	r9d, 11
12823        vmovdqu	xmm7, OWORD PTR [rsi+160]
12824        jl	L_AES_GCM_encrypt_avx2_aesenc_128_enc_done
12825        vaesenc	xmm8, xmm8, xmm7
12826        vaesenc	xmm9, xmm9, xmm7
12827        vaesenc	xmm10, xmm10, xmm7
12828        vaesenc	xmm11, xmm11, xmm7
12829        vaesenc	xmm12, xmm12, xmm7
12830        vaesenc	xmm13, xmm13, xmm7
12831        vaesenc	xmm14, xmm14, xmm7
12832        vaesenc	xmm15, xmm15, xmm7
12833        vmovdqu	xmm7, OWORD PTR [rsi+176]
12834        vaesenc	xmm8, xmm8, xmm7
12835        vaesenc	xmm9, xmm9, xmm7
12836        vaesenc	xmm10, xmm10, xmm7
12837        vaesenc	xmm11, xmm11, xmm7
12838        vaesenc	xmm12, xmm12, xmm7
12839        vaesenc	xmm13, xmm13, xmm7
12840        vaesenc	xmm14, xmm14, xmm7
12841        vaesenc	xmm15, xmm15, xmm7
12842        cmp	r9d, 13
12843        vmovdqu	xmm7, OWORD PTR [rsi+192]
12844        jl	L_AES_GCM_encrypt_avx2_aesenc_128_enc_done
12845        vaesenc	xmm8, xmm8, xmm7
12846        vaesenc	xmm9, xmm9, xmm7
12847        vaesenc	xmm10, xmm10, xmm7
12848        vaesenc	xmm11, xmm11, xmm7
12849        vaesenc	xmm12, xmm12, xmm7
12850        vaesenc	xmm13, xmm13, xmm7
12851        vaesenc	xmm14, xmm14, xmm7
12852        vaesenc	xmm15, xmm15, xmm7
12853        vmovdqu	xmm7, OWORD PTR [rsi+208]
12854        vaesenc	xmm8, xmm8, xmm7
12855        vaesenc	xmm9, xmm9, xmm7
12856        vaesenc	xmm10, xmm10, xmm7
12857        vaesenc	xmm11, xmm11, xmm7
12858        vaesenc	xmm12, xmm12, xmm7
12859        vaesenc	xmm13, xmm13, xmm7
12860        vaesenc	xmm14, xmm14, xmm7
12861        vaesenc	xmm15, xmm15, xmm7
12862        vmovdqu	xmm7, OWORD PTR [rsi+224]
12863L_AES_GCM_encrypt_avx2_aesenc_128_enc_done:
12864        ; aesenc_last
12865        vaesenclast	xmm8, xmm8, xmm7
12866        vaesenclast	xmm9, xmm9, xmm7
12867        vaesenclast	xmm10, xmm10, xmm7
12868        vaesenclast	xmm11, xmm11, xmm7
12869        vmovdqu	xmm0, OWORD PTR [rdi]
12870        vmovdqu	xmm1, OWORD PTR [rdi+16]
12871        vmovdqu	xmm2, OWORD PTR [rdi+32]
12872        vmovdqu	xmm3, OWORD PTR [rdi+48]
12873        vpxor	xmm8, xmm8, xmm0
12874        vpxor	xmm9, xmm9, xmm1
12875        vpxor	xmm10, xmm10, xmm2
12876        vpxor	xmm11, xmm11, xmm3
12877        vmovdqu	OWORD PTR [r8], xmm8
12878        vmovdqu	OWORD PTR [r8+16], xmm9
12879        vmovdqu	OWORD PTR [r8+32], xmm10
12880        vmovdqu	OWORD PTR [r8+48], xmm11
12881        vaesenclast	xmm12, xmm12, xmm7
12882        vaesenclast	xmm13, xmm13, xmm7
12883        vaesenclast	xmm14, xmm14, xmm7
12884        vaesenclast	xmm15, xmm15, xmm7
12885        vmovdqu	xmm0, OWORD PTR [rdi+64]
12886        vmovdqu	xmm1, OWORD PTR [rdi+80]
12887        vmovdqu	xmm2, OWORD PTR [rdi+96]
12888        vmovdqu	xmm3, OWORD PTR [rdi+112]
12889        vpxor	xmm12, xmm12, xmm0
12890        vpxor	xmm13, xmm13, xmm1
12891        vpxor	xmm14, xmm14, xmm2
12892        vpxor	xmm15, xmm15, xmm3
12893        vmovdqu	OWORD PTR [r8+64], xmm12
12894        vmovdqu	OWORD PTR [r8+80], xmm13
12895        vmovdqu	OWORD PTR [r8+96], xmm14
12896        vmovdqu	OWORD PTR [r8+112], xmm15
12897        cmp	r13d, 128
12898        mov	ebx, 128
12899        jle	L_AES_GCM_encrypt_avx2_end_128
12900        ; More 128 bytes of input
12901L_AES_GCM_encrypt_avx2_ghash_128:
12902        ; aesenc_128_ghash
12903        lea	rcx, QWORD PTR [rdi+rbx]
12904        lea	rdx, QWORD PTR [r8+rbx]
12905        ; aesenc_ctr
12906        vmovdqu	xmm0, OWORD PTR [rsp+128]
12907        vmovdqu	xmm1, OWORD PTR L_avx2_aes_gcm_bswap_epi64
12908        vpaddd	xmm9, xmm0, OWORD PTR L_avx2_aes_gcm_one
12909        vpshufb	xmm8, xmm0, xmm1
12910        vpaddd	xmm10, xmm0, OWORD PTR L_avx2_aes_gcm_two
12911        vpshufb	xmm9, xmm9, xmm1
12912        vpaddd	xmm11, xmm0, OWORD PTR L_avx2_aes_gcm_three
12913        vpshufb	xmm10, xmm10, xmm1
12914        vpaddd	xmm12, xmm0, OWORD PTR L_avx2_aes_gcm_four
12915        vpshufb	xmm11, xmm11, xmm1
12916        vpaddd	xmm13, xmm0, OWORD PTR L_avx2_aes_gcm_five
12917        vpshufb	xmm12, xmm12, xmm1
12918        vpaddd	xmm14, xmm0, OWORD PTR L_avx2_aes_gcm_six
12919        vpshufb	xmm13, xmm13, xmm1
12920        vpaddd	xmm15, xmm0, OWORD PTR L_avx2_aes_gcm_seven
12921        vpshufb	xmm14, xmm14, xmm1
12922        vpaddd	xmm0, xmm0, OWORD PTR L_avx2_aes_gcm_eight
12923        vpshufb	xmm15, xmm15, xmm1
12924        ; aesenc_xor
12925        vmovdqu	xmm7, OWORD PTR [rsi]
12926        vmovdqu	OWORD PTR [rsp+128], xmm0
12927        vpxor	xmm8, xmm8, xmm7
12928        vpxor	xmm9, xmm9, xmm7
12929        vpxor	xmm10, xmm10, xmm7
12930        vpxor	xmm11, xmm11, xmm7
12931        vpxor	xmm12, xmm12, xmm7
12932        vpxor	xmm13, xmm13, xmm7
12933        vpxor	xmm14, xmm14, xmm7
12934        vpxor	xmm15, xmm15, xmm7
12935        ; aesenc_pclmul_1
12936        vmovdqu	xmm1, OWORD PTR [rdx+-128]
12937        vmovdqu	xmm0, OWORD PTR [rsi+16]
12938        vpshufb	xmm1, xmm1, OWORD PTR L_avx2_aes_gcm_bswap_mask
12939        vmovdqu	xmm2, OWORD PTR [rsp+112]
12940        vpxor	xmm1, xmm1, xmm6
12941        vpclmulqdq	xmm5, xmm1, xmm2, 16
12942        vpclmulqdq	xmm3, xmm1, xmm2, 1
12943        vpclmulqdq	xmm6, xmm1, xmm2, 0
12944        vpclmulqdq	xmm7, xmm1, xmm2, 17
12945        vaesenc	xmm8, xmm8, xmm0
12946        vaesenc	xmm9, xmm9, xmm0
12947        vaesenc	xmm10, xmm10, xmm0
12948        vaesenc	xmm11, xmm11, xmm0
12949        vaesenc	xmm12, xmm12, xmm0
12950        vaesenc	xmm13, xmm13, xmm0
12951        vaesenc	xmm14, xmm14, xmm0
12952        vaesenc	xmm15, xmm15, xmm0
12953        ; aesenc_pclmul_2
12954        vmovdqu	xmm1, OWORD PTR [rdx+-112]
12955        vmovdqu	xmm0, OWORD PTR [rsp+96]
12956        vpshufb	xmm1, xmm1, OWORD PTR L_avx2_aes_gcm_bswap_mask
12957        vpxor	xmm5, xmm5, xmm3
12958        vpclmulqdq	xmm2, xmm1, xmm0, 16
12959        vpclmulqdq	xmm3, xmm1, xmm0, 1
12960        vpclmulqdq	xmm4, xmm1, xmm0, 0
12961        vpclmulqdq	xmm1, xmm1, xmm0, 17
12962        vmovdqu	xmm0, OWORD PTR [rsi+32]
12963        vpxor	xmm7, xmm7, xmm1
12964        vaesenc	xmm8, xmm8, xmm0
12965        vaesenc	xmm9, xmm9, xmm0
12966        vaesenc	xmm10, xmm10, xmm0
12967        vaesenc	xmm11, xmm11, xmm0
12968        vaesenc	xmm12, xmm12, xmm0
12969        vaesenc	xmm13, xmm13, xmm0
12970        vaesenc	xmm14, xmm14, xmm0
12971        vaesenc	xmm15, xmm15, xmm0
12972        ; aesenc_pclmul_n
12973        vmovdqu	xmm1, OWORD PTR [rdx+-96]
12974        vmovdqu	xmm0, OWORD PTR [rsp+80]
12975        vpshufb	xmm1, xmm1, OWORD PTR L_avx2_aes_gcm_bswap_mask
12976        vpxor	xmm5, xmm5, xmm2
12977        vpclmulqdq	xmm2, xmm1, xmm0, 16
12978        vpxor	xmm5, xmm5, xmm3
12979        vpclmulqdq	xmm3, xmm1, xmm0, 1
12980        vpxor	xmm6, xmm6, xmm4
12981        vpclmulqdq	xmm4, xmm1, xmm0, 0
12982        vpclmulqdq	xmm1, xmm1, xmm0, 17
12983        vmovdqu	xmm0, OWORD PTR [rsi+48]
12984        vpxor	xmm7, xmm7, xmm1
12985        vaesenc	xmm8, xmm8, xmm0
12986        vaesenc	xmm9, xmm9, xmm0
12987        vaesenc	xmm10, xmm10, xmm0
12988        vaesenc	xmm11, xmm11, xmm0
12989        vaesenc	xmm12, xmm12, xmm0
12990        vaesenc	xmm13, xmm13, xmm0
12991        vaesenc	xmm14, xmm14, xmm0
12992        vaesenc	xmm15, xmm15, xmm0
12993        ; aesenc_pclmul_n
12994        vmovdqu	xmm1, OWORD PTR [rdx+-80]
12995        vmovdqu	xmm0, OWORD PTR [rsp+64]
12996        vpshufb	xmm1, xmm1, OWORD PTR L_avx2_aes_gcm_bswap_mask
12997        vpxor	xmm5, xmm5, xmm2
12998        vpclmulqdq	xmm2, xmm1, xmm0, 16
12999        vpxor	xmm5, xmm5, xmm3
13000        vpclmulqdq	xmm3, xmm1, xmm0, 1
13001        vpxor	xmm6, xmm6, xmm4
13002        vpclmulqdq	xmm4, xmm1, xmm0, 0
13003        vpclmulqdq	xmm1, xmm1, xmm0, 17
13004        vmovdqu	xmm0, OWORD PTR [rsi+64]
13005        vpxor	xmm7, xmm7, xmm1
13006        vaesenc	xmm8, xmm8, xmm0
13007        vaesenc	xmm9, xmm9, xmm0
13008        vaesenc	xmm10, xmm10, xmm0
13009        vaesenc	xmm11, xmm11, xmm0
13010        vaesenc	xmm12, xmm12, xmm0
13011        vaesenc	xmm13, xmm13, xmm0
13012        vaesenc	xmm14, xmm14, xmm0
13013        vaesenc	xmm15, xmm15, xmm0
13014        ; aesenc_pclmul_n
13015        vmovdqu	xmm1, OWORD PTR [rdx+-64]
13016        vmovdqu	xmm0, OWORD PTR [rsp+48]
13017        vpshufb	xmm1, xmm1, OWORD PTR L_avx2_aes_gcm_bswap_mask
13018        vpxor	xmm5, xmm5, xmm2
13019        vpclmulqdq	xmm2, xmm1, xmm0, 16
13020        vpxor	xmm5, xmm5, xmm3
13021        vpclmulqdq	xmm3, xmm1, xmm0, 1
13022        vpxor	xmm6, xmm6, xmm4
13023        vpclmulqdq	xmm4, xmm1, xmm0, 0
13024        vpclmulqdq	xmm1, xmm1, xmm0, 17
13025        vmovdqu	xmm0, OWORD PTR [rsi+80]
13026        vpxor	xmm7, xmm7, xmm1
13027        vaesenc	xmm8, xmm8, xmm0
13028        vaesenc	xmm9, xmm9, xmm0
13029        vaesenc	xmm10, xmm10, xmm0
13030        vaesenc	xmm11, xmm11, xmm0
13031        vaesenc	xmm12, xmm12, xmm0
13032        vaesenc	xmm13, xmm13, xmm0
13033        vaesenc	xmm14, xmm14, xmm0
13034        vaesenc	xmm15, xmm15, xmm0
13035        ; aesenc_pclmul_n
13036        vmovdqu	xmm1, OWORD PTR [rdx+-48]
13037        vmovdqu	xmm0, OWORD PTR [rsp+32]
13038        vpshufb	xmm1, xmm1, OWORD PTR L_avx2_aes_gcm_bswap_mask
13039        vpxor	xmm5, xmm5, xmm2
13040        vpclmulqdq	xmm2, xmm1, xmm0, 16
13041        vpxor	xmm5, xmm5, xmm3
13042        vpclmulqdq	xmm3, xmm1, xmm0, 1
13043        vpxor	xmm6, xmm6, xmm4
13044        vpclmulqdq	xmm4, xmm1, xmm0, 0
13045        vpclmulqdq	xmm1, xmm1, xmm0, 17
13046        vmovdqu	xmm0, OWORD PTR [rsi+96]
13047        vpxor	xmm7, xmm7, xmm1
13048        vaesenc	xmm8, xmm8, xmm0
13049        vaesenc	xmm9, xmm9, xmm0
13050        vaesenc	xmm10, xmm10, xmm0
13051        vaesenc	xmm11, xmm11, xmm0
13052        vaesenc	xmm12, xmm12, xmm0
13053        vaesenc	xmm13, xmm13, xmm0
13054        vaesenc	xmm14, xmm14, xmm0
13055        vaesenc	xmm15, xmm15, xmm0
13056        ; aesenc_pclmul_n
13057        vmovdqu	xmm1, OWORD PTR [rdx+-32]
13058        vmovdqu	xmm0, OWORD PTR [rsp+16]
13059        vpshufb	xmm1, xmm1, OWORD PTR L_avx2_aes_gcm_bswap_mask
13060        vpxor	xmm5, xmm5, xmm2
13061        vpclmulqdq	xmm2, xmm1, xmm0, 16
13062        vpxor	xmm5, xmm5, xmm3
13063        vpclmulqdq	xmm3, xmm1, xmm0, 1
13064        vpxor	xmm6, xmm6, xmm4
13065        vpclmulqdq	xmm4, xmm1, xmm0, 0
13066        vpclmulqdq	xmm1, xmm1, xmm0, 17
13067        vmovdqu	xmm0, OWORD PTR [rsi+112]
13068        vpxor	xmm7, xmm7, xmm1
13069        vaesenc	xmm8, xmm8, xmm0
13070        vaesenc	xmm9, xmm9, xmm0
13071        vaesenc	xmm10, xmm10, xmm0
13072        vaesenc	xmm11, xmm11, xmm0
13073        vaesenc	xmm12, xmm12, xmm0
13074        vaesenc	xmm13, xmm13, xmm0
13075        vaesenc	xmm14, xmm14, xmm0
13076        vaesenc	xmm15, xmm15, xmm0
13077        ; aesenc_pclmul_n
13078        vmovdqu	xmm1, OWORD PTR [rdx+-16]
13079        vmovdqu	xmm0, OWORD PTR [rsp]
13080        vpshufb	xmm1, xmm1, OWORD PTR L_avx2_aes_gcm_bswap_mask
13081        vpxor	xmm5, xmm5, xmm2
13082        vpclmulqdq	xmm2, xmm1, xmm0, 16
13083        vpxor	xmm5, xmm5, xmm3
13084        vpclmulqdq	xmm3, xmm1, xmm0, 1
13085        vpxor	xmm6, xmm6, xmm4
13086        vpclmulqdq	xmm4, xmm1, xmm0, 0
13087        vpclmulqdq	xmm1, xmm1, xmm0, 17
13088        vmovdqu	xmm0, OWORD PTR [rsi+128]
13089        vpxor	xmm7, xmm7, xmm1
13090        vaesenc	xmm8, xmm8, xmm0
13091        vaesenc	xmm9, xmm9, xmm0
13092        vaesenc	xmm10, xmm10, xmm0
13093        vaesenc	xmm11, xmm11, xmm0
13094        vaesenc	xmm12, xmm12, xmm0
13095        vaesenc	xmm13, xmm13, xmm0
13096        vaesenc	xmm14, xmm14, xmm0
13097        vaesenc	xmm15, xmm15, xmm0
13098        ; aesenc_pclmul_l
13099        vpxor	xmm5, xmm5, xmm2
13100        vpxor	xmm6, xmm6, xmm4
13101        vpxor	xmm5, xmm5, xmm3
13102        vpslldq	xmm1, xmm5, 8
13103        vpsrldq	xmm5, xmm5, 8
13104        vmovdqu	xmm4, OWORD PTR [rsi+144]
13105        vmovdqu	xmm0, OWORD PTR L_avx2_aes_gcm_mod2_128
13106        vaesenc	xmm8, xmm8, xmm4
13107        vpxor	xmm6, xmm6, xmm1
13108        vpxor	xmm7, xmm7, xmm5
13109        vpclmulqdq	xmm3, xmm6, xmm0, 16
13110        vaesenc	xmm9, xmm9, xmm4
13111        vaesenc	xmm10, xmm10, xmm4
13112        vaesenc	xmm11, xmm11, xmm4
13113        vpshufd	xmm6, xmm6, 78
13114        vpxor	xmm6, xmm6, xmm3
13115        vpclmulqdq	xmm3, xmm6, xmm0, 16
13116        vaesenc	xmm12, xmm12, xmm4
13117        vaesenc	xmm13, xmm13, xmm4
13118        vaesenc	xmm14, xmm14, xmm4
13119        vpshufd	xmm6, xmm6, 78
13120        vpxor	xmm6, xmm6, xmm3
13121        vpxor	xmm6, xmm6, xmm7
13122        vaesenc	xmm15, xmm15, xmm4
13123        cmp	r9d, 11
13124        vmovdqu	xmm7, OWORD PTR [rsi+160]
13125        jl	L_AES_GCM_encrypt_avx2_aesenc_128_ghash_avx_done
13126        vaesenc	xmm8, xmm8, xmm7
13127        vaesenc	xmm9, xmm9, xmm7
13128        vaesenc	xmm10, xmm10, xmm7
13129        vaesenc	xmm11, xmm11, xmm7
13130        vaesenc	xmm12, xmm12, xmm7
13131        vaesenc	xmm13, xmm13, xmm7
13132        vaesenc	xmm14, xmm14, xmm7
13133        vaesenc	xmm15, xmm15, xmm7
13134        vmovdqu	xmm7, OWORD PTR [rsi+176]
13135        vaesenc	xmm8, xmm8, xmm7
13136        vaesenc	xmm9, xmm9, xmm7
13137        vaesenc	xmm10, xmm10, xmm7
13138        vaesenc	xmm11, xmm11, xmm7
13139        vaesenc	xmm12, xmm12, xmm7
13140        vaesenc	xmm13, xmm13, xmm7
13141        vaesenc	xmm14, xmm14, xmm7
13142        vaesenc	xmm15, xmm15, xmm7
13143        cmp	r9d, 13
13144        vmovdqu	xmm7, OWORD PTR [rsi+192]
13145        jl	L_AES_GCM_encrypt_avx2_aesenc_128_ghash_avx_done
13146        vaesenc	xmm8, xmm8, xmm7
13147        vaesenc	xmm9, xmm9, xmm7
13148        vaesenc	xmm10, xmm10, xmm7
13149        vaesenc	xmm11, xmm11, xmm7
13150        vaesenc	xmm12, xmm12, xmm7
13151        vaesenc	xmm13, xmm13, xmm7
13152        vaesenc	xmm14, xmm14, xmm7
13153        vaesenc	xmm15, xmm15, xmm7
13154        vmovdqu	xmm7, OWORD PTR [rsi+208]
13155        vaesenc	xmm8, xmm8, xmm7
13156        vaesenc	xmm9, xmm9, xmm7
13157        vaesenc	xmm10, xmm10, xmm7
13158        vaesenc	xmm11, xmm11, xmm7
13159        vaesenc	xmm12, xmm12, xmm7
13160        vaesenc	xmm13, xmm13, xmm7
13161        vaesenc	xmm14, xmm14, xmm7
13162        vaesenc	xmm15, xmm15, xmm7
13163        vmovdqu	xmm7, OWORD PTR [rsi+224]
13164L_AES_GCM_encrypt_avx2_aesenc_128_ghash_avx_done:
13165        ; aesenc_last
13166        vaesenclast	xmm8, xmm8, xmm7
13167        vaesenclast	xmm9, xmm9, xmm7
13168        vaesenclast	xmm10, xmm10, xmm7
13169        vaesenclast	xmm11, xmm11, xmm7
13170        vmovdqu	xmm0, OWORD PTR [rcx]
13171        vmovdqu	xmm1, OWORD PTR [rcx+16]
13172        vmovdqu	xmm2, OWORD PTR [rcx+32]
13173        vmovdqu	xmm3, OWORD PTR [rcx+48]
13174        vpxor	xmm8, xmm8, xmm0
13175        vpxor	xmm9, xmm9, xmm1
13176        vpxor	xmm10, xmm10, xmm2
13177        vpxor	xmm11, xmm11, xmm3
13178        vmovdqu	OWORD PTR [rdx], xmm8
13179        vmovdqu	OWORD PTR [rdx+16], xmm9
13180        vmovdqu	OWORD PTR [rdx+32], xmm10
13181        vmovdqu	OWORD PTR [rdx+48], xmm11
13182        vaesenclast	xmm12, xmm12, xmm7
13183        vaesenclast	xmm13, xmm13, xmm7
13184        vaesenclast	xmm14, xmm14, xmm7
13185        vaesenclast	xmm15, xmm15, xmm7
13186        vmovdqu	xmm0, OWORD PTR [rcx+64]
13187        vmovdqu	xmm1, OWORD PTR [rcx+80]
13188        vmovdqu	xmm2, OWORD PTR [rcx+96]
13189        vmovdqu	xmm3, OWORD PTR [rcx+112]
13190        vpxor	xmm12, xmm12, xmm0
13191        vpxor	xmm13, xmm13, xmm1
13192        vpxor	xmm14, xmm14, xmm2
13193        vpxor	xmm15, xmm15, xmm3
13194        vmovdqu	OWORD PTR [rdx+64], xmm12
13195        vmovdqu	OWORD PTR [rdx+80], xmm13
13196        vmovdqu	OWORD PTR [rdx+96], xmm14
13197        vmovdqu	OWORD PTR [rdx+112], xmm15
13198        ; aesenc_128_ghash - end
13199        add	ebx, 128
13200        cmp	ebx, r13d
13201        jl	L_AES_GCM_encrypt_avx2_ghash_128
13202L_AES_GCM_encrypt_avx2_end_128:
13203        vmovdqu	xmm4, OWORD PTR L_avx2_aes_gcm_bswap_mask
13204        vpshufb	xmm8, xmm8, xmm4
13205        vpshufb	xmm9, xmm9, xmm4
13206        vpshufb	xmm10, xmm10, xmm4
13207        vpshufb	xmm11, xmm11, xmm4
13208        vpshufb	xmm12, xmm12, xmm4
13209        vpshufb	xmm13, xmm13, xmm4
13210        vpshufb	xmm14, xmm14, xmm4
13211        vpshufb	xmm15, xmm15, xmm4
13212        vpxor	xmm8, xmm8, xmm6
13213        vmovdqu	xmm7, OWORD PTR [rsp]
13214        vpclmulqdq	xmm5, xmm7, xmm15, 16
13215        vpclmulqdq	xmm1, xmm7, xmm15, 1
13216        vpclmulqdq	xmm4, xmm7, xmm15, 0
13217        vpclmulqdq	xmm6, xmm7, xmm15, 17
13218        vpxor	xmm5, xmm5, xmm1
13219        vmovdqu	xmm7, OWORD PTR [rsp+16]
13220        vpclmulqdq	xmm2, xmm7, xmm14, 16
13221        vpclmulqdq	xmm1, xmm7, xmm14, 1
13222        vpclmulqdq	xmm0, xmm7, xmm14, 0
13223        vpclmulqdq	xmm3, xmm7, xmm14, 17
13224        vpxor	xmm2, xmm2, xmm1
13225        vpxor	xmm6, xmm6, xmm3
13226        vpxor	xmm5, xmm5, xmm2
13227        vpxor	xmm4, xmm4, xmm0
13228        vmovdqu	xmm15, OWORD PTR [rsp+32]
13229        vmovdqu	xmm7, OWORD PTR [rsp+48]
13230        vpclmulqdq	xmm2, xmm15, xmm13, 16
13231        vpclmulqdq	xmm1, xmm15, xmm13, 1
13232        vpclmulqdq	xmm0, xmm15, xmm13, 0
13233        vpclmulqdq	xmm3, xmm15, xmm13, 17
13234        vpxor	xmm2, xmm2, xmm1
13235        vpxor	xmm6, xmm6, xmm3
13236        vpxor	xmm5, xmm5, xmm2
13237        vpxor	xmm4, xmm4, xmm0
13238        vpclmulqdq	xmm2, xmm7, xmm12, 16
13239        vpclmulqdq	xmm1, xmm7, xmm12, 1
13240        vpclmulqdq	xmm0, xmm7, xmm12, 0
13241        vpclmulqdq	xmm3, xmm7, xmm12, 17
13242        vpxor	xmm2, xmm2, xmm1
13243        vpxor	xmm6, xmm6, xmm3
13244        vpxor	xmm5, xmm5, xmm2
13245        vpxor	xmm4, xmm4, xmm0
13246        vmovdqu	xmm15, OWORD PTR [rsp+64]
13247        vmovdqu	xmm7, OWORD PTR [rsp+80]
13248        vpclmulqdq	xmm2, xmm15, xmm11, 16
13249        vpclmulqdq	xmm1, xmm15, xmm11, 1
13250        vpclmulqdq	xmm0, xmm15, xmm11, 0
13251        vpclmulqdq	xmm3, xmm15, xmm11, 17
13252        vpxor	xmm2, xmm2, xmm1
13253        vpxor	xmm6, xmm6, xmm3
13254        vpxor	xmm5, xmm5, xmm2
13255        vpxor	xmm4, xmm4, xmm0
13256        vpclmulqdq	xmm2, xmm7, xmm10, 16
13257        vpclmulqdq	xmm1, xmm7, xmm10, 1
13258        vpclmulqdq	xmm0, xmm7, xmm10, 0
13259        vpclmulqdq	xmm3, xmm7, xmm10, 17
13260        vpxor	xmm2, xmm2, xmm1
13261        vpxor	xmm6, xmm6, xmm3
13262        vpxor	xmm5, xmm5, xmm2
13263        vpxor	xmm4, xmm4, xmm0
13264        vmovdqu	xmm15, OWORD PTR [rsp+96]
13265        vmovdqu	xmm7, OWORD PTR [rsp+112]
13266        vpclmulqdq	xmm2, xmm15, xmm9, 16
13267        vpclmulqdq	xmm1, xmm15, xmm9, 1
13268        vpclmulqdq	xmm0, xmm15, xmm9, 0
13269        vpclmulqdq	xmm3, xmm15, xmm9, 17
13270        vpxor	xmm2, xmm2, xmm1
13271        vpxor	xmm6, xmm6, xmm3
13272        vpxor	xmm5, xmm5, xmm2
13273        vpxor	xmm4, xmm4, xmm0
13274        vpclmulqdq	xmm2, xmm7, xmm8, 16
13275        vpclmulqdq	xmm1, xmm7, xmm8, 1
13276        vpclmulqdq	xmm0, xmm7, xmm8, 0
13277        vpclmulqdq	xmm3, xmm7, xmm8, 17
13278        vpxor	xmm2, xmm2, xmm1
13279        vpxor	xmm6, xmm6, xmm3
13280        vpxor	xmm5, xmm5, xmm2
13281        vpxor	xmm4, xmm4, xmm0
13282        vpslldq	xmm7, xmm5, 8
13283        vpsrldq	xmm5, xmm5, 8
13284        vpxor	xmm4, xmm4, xmm7
13285        vpxor	xmm6, xmm6, xmm5
13286        ; ghash_red
13287        vmovdqu	xmm2, OWORD PTR L_avx2_aes_gcm_mod2_128
13288        vpclmulqdq	xmm0, xmm4, xmm2, 16
13289        vpshufd	xmm1, xmm4, 78
13290        vpxor	xmm1, xmm1, xmm0
13291        vpclmulqdq	xmm0, xmm1, xmm2, 16
13292        vpshufd	xmm1, xmm1, 78
13293        vpxor	xmm1, xmm1, xmm0
13294        vpxor	xmm6, xmm6, xmm1
13295        vmovdqu	xmm5, OWORD PTR [rsp]
13296        vmovdqu	xmm4, OWORD PTR [rsp+128]
13297        vmovdqu	xmm15, OWORD PTR [rsp+144]
13298L_AES_GCM_encrypt_avx2_done_128:
13299        cmp	ebx, r10d
13300        je	L_AES_GCM_encrypt_avx2_done_enc
13301        mov	r13d, r10d
13302        and	r13d, 4294967280
13303        cmp	ebx, r13d
13304        jge	L_AES_GCM_encrypt_avx2_last_block_done
13305        ; aesenc_block
13306        vmovdqu	xmm1, xmm4
13307        vpshufb	xmm0, xmm1, OWORD PTR L_avx2_aes_gcm_bswap_epi64
13308        vpaddd	xmm1, xmm1, OWORD PTR L_avx2_aes_gcm_one
13309        vpxor	xmm0, xmm0, [rsi]
13310        vmovdqu	xmm2, OWORD PTR [rsi+16]
13311        vaesenc	xmm0, xmm0, xmm2
13312        vmovdqu	xmm2, OWORD PTR [rsi+32]
13313        vaesenc	xmm0, xmm0, xmm2
13314        vmovdqu	xmm2, OWORD PTR [rsi+48]
13315        vaesenc	xmm0, xmm0, xmm2
13316        vmovdqu	xmm2, OWORD PTR [rsi+64]
13317        vaesenc	xmm0, xmm0, xmm2
13318        vmovdqu	xmm2, OWORD PTR [rsi+80]
13319        vaesenc	xmm0, xmm0, xmm2
13320        vmovdqu	xmm2, OWORD PTR [rsi+96]
13321        vaesenc	xmm0, xmm0, xmm2
13322        vmovdqu	xmm2, OWORD PTR [rsi+112]
13323        vaesenc	xmm0, xmm0, xmm2
13324        vmovdqu	xmm2, OWORD PTR [rsi+128]
13325        vaesenc	xmm0, xmm0, xmm2
13326        vmovdqu	xmm2, OWORD PTR [rsi+144]
13327        vaesenc	xmm0, xmm0, xmm2
13328        vmovdqu	xmm4, xmm1
13329        cmp	r9d, 11
13330        vmovdqu	xmm1, OWORD PTR [rsi+160]
13331        jl	L_AES_GCM_encrypt_avx2_aesenc_block_last
13332        vaesenc	xmm0, xmm0, xmm1
13333        vmovdqu	xmm2, OWORD PTR [rsi+176]
13334        vaesenc	xmm0, xmm0, xmm2
13335        cmp	r9d, 13
13336        vmovdqu	xmm1, OWORD PTR [rsi+192]
13337        jl	L_AES_GCM_encrypt_avx2_aesenc_block_last
13338        vaesenc	xmm0, xmm0, xmm1
13339        vmovdqu	xmm2, OWORD PTR [rsi+208]
13340        vaesenc	xmm0, xmm0, xmm2
13341        vmovdqu	xmm1, OWORD PTR [rsi+224]
13342L_AES_GCM_encrypt_avx2_aesenc_block_last:
13343        vaesenclast	xmm0, xmm0, xmm1
13344        vmovdqu	xmm1, OWORD PTR [rdi+rbx]
13345        vpxor	xmm0, xmm0, xmm1
13346        vmovdqu	OWORD PTR [r8+rbx], xmm0
13347        vpshufb	xmm0, xmm0, OWORD PTR L_avx2_aes_gcm_bswap_mask
13348        vpxor	xmm6, xmm6, xmm0
13349        add	ebx, 16
13350        cmp	ebx, r13d
13351        jge	L_AES_GCM_encrypt_avx2_last_block_ghash
13352L_AES_GCM_encrypt_avx2_last_block_start:
13353        vmovdqu	xmm12, OWORD PTR [rdi+rbx]
13354        vpshufb	xmm11, xmm4, OWORD PTR L_avx2_aes_gcm_bswap_epi64
13355        vpaddd	xmm4, xmm4, OWORD PTR L_avx2_aes_gcm_one
13356        ; aesenc_gfmul_sb
13357        vpclmulqdq	xmm2, xmm6, xmm5, 1
13358        vpclmulqdq	xmm3, xmm6, xmm5, 16
13359        vpclmulqdq	xmm1, xmm6, xmm5, 0
13360        vpclmulqdq	xmm8, xmm6, xmm5, 17
13361        vpxor	xmm11, xmm11, [rsi]
13362        vaesenc	xmm11, xmm11, [rsi+16]
13363        vpxor	xmm3, xmm3, xmm2
13364        vpslldq	xmm2, xmm3, 8
13365        vpsrldq	xmm3, xmm3, 8
13366        vaesenc	xmm11, xmm11, [rsi+32]
13367        vpxor	xmm2, xmm2, xmm1
13368        vpclmulqdq	xmm1, xmm2, OWORD PTR L_avx2_aes_gcm_mod2_128, 16
13369        vaesenc	xmm11, xmm11, [rsi+48]
13370        vaesenc	xmm11, xmm11, [rsi+64]
13371        vaesenc	xmm11, xmm11, [rsi+80]
13372        vpshufd	xmm2, xmm2, 78
13373        vpxor	xmm2, xmm2, xmm1
13374        vpclmulqdq	xmm1, xmm2, OWORD PTR L_avx2_aes_gcm_mod2_128, 16
13375        vaesenc	xmm11, xmm11, [rsi+96]
13376        vaesenc	xmm11, xmm11, [rsi+112]
13377        vaesenc	xmm11, xmm11, [rsi+128]
13378        vpshufd	xmm2, xmm2, 78
13379        vaesenc	xmm11, xmm11, [rsi+144]
13380        vpxor	xmm8, xmm8, xmm3
13381        vpxor	xmm2, xmm2, xmm8
13382        vmovdqu	xmm0, OWORD PTR [rsi+160]
13383        cmp	r9d, 11
13384        jl	L_AES_GCM_encrypt_avx2_aesenc_gfmul_sb_last
13385        vaesenc	xmm11, xmm11, xmm0
13386        vaesenc	xmm11, xmm11, [rsi+176]
13387        vmovdqu	xmm0, OWORD PTR [rsi+192]
13388        cmp	r9d, 13
13389        jl	L_AES_GCM_encrypt_avx2_aesenc_gfmul_sb_last
13390        vaesenc	xmm11, xmm11, xmm0
13391        vaesenc	xmm11, xmm11, [rsi+208]
13392        vmovdqu	xmm0, OWORD PTR [rsi+224]
13393L_AES_GCM_encrypt_avx2_aesenc_gfmul_sb_last:
13394        vaesenclast	xmm11, xmm11, xmm0
13395        vpxor	xmm6, xmm2, xmm1
13396        vpxor	xmm11, xmm11, xmm12
13397        vmovdqu	OWORD PTR [r8+rbx], xmm11
13398        vpshufb	xmm11, xmm11, OWORD PTR L_avx2_aes_gcm_bswap_mask
13399        vpxor	xmm6, xmm6, xmm11
13400        add	ebx, 16
13401        cmp	ebx, r13d
13402        jl	L_AES_GCM_encrypt_avx2_last_block_start
13403L_AES_GCM_encrypt_avx2_last_block_ghash:
13404        ; ghash_gfmul_red
13405        vpclmulqdq	xmm10, xmm6, xmm5, 16
13406        vpclmulqdq	xmm9, xmm6, xmm5, 1
13407        vpclmulqdq	xmm8, xmm6, xmm5, 0
13408        vpxor	xmm10, xmm10, xmm9
13409        vpslldq	xmm9, xmm10, 8
13410        vpsrldq	xmm10, xmm10, 8
13411        vpxor	xmm9, xmm9, xmm8
13412        vpclmulqdq	xmm6, xmm6, xmm5, 17
13413        vpclmulqdq	xmm8, xmm9, OWORD PTR L_avx2_aes_gcm_mod2_128, 16
13414        vpshufd	xmm9, xmm9, 78
13415        vpxor	xmm9, xmm9, xmm8
13416        vpclmulqdq	xmm8, xmm9, OWORD PTR L_avx2_aes_gcm_mod2_128, 16
13417        vpshufd	xmm9, xmm9, 78
13418        vpxor	xmm6, xmm6, xmm10
13419        vpxor	xmm6, xmm6, xmm9
13420        vpxor	xmm6, xmm6, xmm8
13421L_AES_GCM_encrypt_avx2_last_block_done:
13422        mov	ecx, r10d
13423        mov	edx, r10d
13424        and	ecx, 15
13425        jz	L_AES_GCM_encrypt_avx2_done_enc
13426        ; aesenc_last15_enc
13427        vpshufb	xmm4, xmm4, OWORD PTR L_avx2_aes_gcm_bswap_epi64
13428        vpxor	xmm4, xmm4, [rsi]
13429        vaesenc	xmm4, xmm4, [rsi+16]
13430        vaesenc	xmm4, xmm4, [rsi+32]
13431        vaesenc	xmm4, xmm4, [rsi+48]
13432        vaesenc	xmm4, xmm4, [rsi+64]
13433        vaesenc	xmm4, xmm4, [rsi+80]
13434        vaesenc	xmm4, xmm4, [rsi+96]
13435        vaesenc	xmm4, xmm4, [rsi+112]
13436        vaesenc	xmm4, xmm4, [rsi+128]
13437        vaesenc	xmm4, xmm4, [rsi+144]
13438        cmp	r9d, 11
13439        vmovdqu	xmm0, OWORD PTR [rsi+160]
13440        jl	L_AES_GCM_encrypt_avx2_aesenc_last15_enc_avx_aesenc_avx_last
13441        vaesenc	xmm4, xmm4, xmm0
13442        vaesenc	xmm4, xmm4, [rsi+176]
13443        cmp	r9d, 13
13444        vmovdqu	xmm0, OWORD PTR [rsi+192]
13445        jl	L_AES_GCM_encrypt_avx2_aesenc_last15_enc_avx_aesenc_avx_last
13446        vaesenc	xmm4, xmm4, xmm0
13447        vaesenc	xmm4, xmm4, [rsi+208]
13448        vmovdqu	xmm0, OWORD PTR [rsi+224]
13449L_AES_GCM_encrypt_avx2_aesenc_last15_enc_avx_aesenc_avx_last:
13450        vaesenclast	xmm4, xmm4, xmm0
13451        xor	ecx, ecx
13452        vpxor	xmm0, xmm0, xmm0
13453        vmovdqu	OWORD PTR [rsp], xmm4
13454        vmovdqu	OWORD PTR [rsp+16], xmm0
13455L_AES_GCM_encrypt_avx2_aesenc_last15_enc_avx_loop:
13456        movzx	r13d, BYTE PTR [rdi+rbx]
13457        xor	r13b, BYTE PTR [rsp+rcx]
13458        mov	BYTE PTR [rsp+rcx+16], r13b
13459        mov	BYTE PTR [r8+rbx], r13b
13460        inc	ebx
13461        inc	ecx
13462        cmp	ebx, edx
13463        jl	L_AES_GCM_encrypt_avx2_aesenc_last15_enc_avx_loop
13464L_AES_GCM_encrypt_avx2_aesenc_last15_enc_avx_finish_enc:
13465        vmovdqu	xmm4, OWORD PTR [rsp+16]
13466        vpshufb	xmm4, xmm4, OWORD PTR L_avx2_aes_gcm_bswap_mask
13467        vpxor	xmm6, xmm6, xmm4
13468        ; ghash_gfmul_red
13469        vpclmulqdq	xmm2, xmm6, xmm5, 16
13470        vpclmulqdq	xmm1, xmm6, xmm5, 1
13471        vpclmulqdq	xmm0, xmm6, xmm5, 0
13472        vpxor	xmm2, xmm2, xmm1
13473        vpslldq	xmm1, xmm2, 8
13474        vpsrldq	xmm2, xmm2, 8
13475        vpxor	xmm1, xmm1, xmm0
13476        vpclmulqdq	xmm6, xmm6, xmm5, 17
13477        vpclmulqdq	xmm0, xmm1, OWORD PTR L_avx2_aes_gcm_mod2_128, 16
13478        vpshufd	xmm1, xmm1, 78
13479        vpxor	xmm1, xmm1, xmm0
13480        vpclmulqdq	xmm0, xmm1, OWORD PTR L_avx2_aes_gcm_mod2_128, 16
13481        vpshufd	xmm1, xmm1, 78
13482        vpxor	xmm6, xmm6, xmm2
13483        vpxor	xmm6, xmm6, xmm1
13484        vpxor	xmm6, xmm6, xmm0
13485L_AES_GCM_encrypt_avx2_done_enc:
13486        ; calc_tag
13487        shl	r10, 3
13488        shl	r11, 3
13489        vmovq	xmm0, r10
13490        vmovq	xmm1, r11
13491        vpunpcklqdq	xmm0, xmm0, xmm1
13492        vpxor	xmm0, xmm0, xmm6
13493        ; ghash_gfmul_red
13494        vpclmulqdq	xmm4, xmm0, xmm5, 16
13495        vpclmulqdq	xmm3, xmm0, xmm5, 1
13496        vpclmulqdq	xmm2, xmm0, xmm5, 0
13497        vpxor	xmm4, xmm4, xmm3
13498        vpslldq	xmm3, xmm4, 8
13499        vpsrldq	xmm4, xmm4, 8
13500        vpxor	xmm3, xmm3, xmm2
13501        vpclmulqdq	xmm0, xmm0, xmm5, 17
13502        vpclmulqdq	xmm2, xmm3, OWORD PTR L_avx2_aes_gcm_mod2_128, 16
13503        vpshufd	xmm3, xmm3, 78
13504        vpxor	xmm3, xmm3, xmm2
13505        vpclmulqdq	xmm2, xmm3, OWORD PTR L_avx2_aes_gcm_mod2_128, 16
13506        vpshufd	xmm3, xmm3, 78
13507        vpxor	xmm0, xmm0, xmm4
13508        vpxor	xmm0, xmm0, xmm3
13509        vpxor	xmm0, xmm0, xmm2
13510        vpshufb	xmm0, xmm0, OWORD PTR L_avx2_aes_gcm_bswap_mask
13511        vpxor	xmm0, xmm0, xmm15
13512        ; store_tag
13513        cmp	r14d, 16
13514        je	L_AES_GCM_encrypt_avx2_store_tag_16
13515        xor	rcx, rcx
13516        vmovdqu	OWORD PTR [rsp], xmm0
13517L_AES_GCM_encrypt_avx2_store_tag_loop:
13518        movzx	r13d, BYTE PTR [rsp+rcx]
13519        mov	BYTE PTR [r15+rcx], r13b
13520        inc	ecx
13521        cmp	ecx, r14d
13522        jne	L_AES_GCM_encrypt_avx2_store_tag_loop
13523        jmp	L_AES_GCM_encrypt_avx2_store_tag_done
13524L_AES_GCM_encrypt_avx2_store_tag_16:
13525        vmovdqu	OWORD PTR [r15], xmm0
13526L_AES_GCM_encrypt_avx2_store_tag_done:
13527        vzeroupper
13528        vmovdqu	xmm6, OWORD PTR [rsp+160]
13529        vmovdqu	xmm7, OWORD PTR [rsp+176]
13530        vmovdqu	xmm8, OWORD PTR [rsp+192]
13531        vmovdqu	xmm9, OWORD PTR [rsp+208]
13532        vmovdqu	xmm10, OWORD PTR [rsp+224]
13533        vmovdqu	xmm11, OWORD PTR [rsp+240]
13534        vmovdqu	xmm12, OWORD PTR [rsp+256]
13535        vmovdqu	xmm13, OWORD PTR [rsp+272]
13536        vmovdqu	xmm14, OWORD PTR [rsp+288]
13537        vmovdqu	xmm15, OWORD PTR [rsp+304]
13538        add	rsp, 320
13539        pop	rsi
13540        pop	r14
13541        pop	rbx
13542        pop	r15
13543        pop	r12
13544        pop	rdi
13545        pop	r13
13546        ret
13547AES_GCM_encrypt_avx2 ENDP
13548_TEXT ENDS
13549_TEXT SEGMENT READONLY PARA
13550AES_GCM_decrypt_avx2 PROC
13551        push	r13
13552        push	rdi
13553        push	r12
13554        push	r14
13555        push	rbx
13556        push	r15
13557        push	rsi
13558        push	rbp
13559        mov	rdi, rcx
13560        mov	r12, r8
13561        mov	rax, r9
13562        mov	r14, QWORD PTR [rsp+104]
13563        mov	r8, rdx
13564        mov	r10d, DWORD PTR [rsp+112]
13565        mov	r11d, DWORD PTR [rsp+120]
13566        mov	ebx, DWORD PTR [rsp+128]
13567        mov	r15d, DWORD PTR [rsp+136]
13568        mov	rsi, QWORD PTR [rsp+144]
13569        mov	r9d, DWORD PTR [rsp+152]
13570        mov	rbp, QWORD PTR [rsp+160]
13571        sub	rsp, 328
13572        vmovdqu	OWORD PTR [rsp+168], xmm6
13573        vmovdqu	OWORD PTR [rsp+184], xmm7
13574        vmovdqu	OWORD PTR [rsp+200], xmm8
13575        vmovdqu	OWORD PTR [rsp+216], xmm9
13576        vmovdqu	OWORD PTR [rsp+232], xmm10
13577        vmovdqu	OWORD PTR [rsp+248], xmm11
13578        vmovdqu	OWORD PTR [rsp+264], xmm12
13579        vmovdqu	OWORD PTR [rsp+280], xmm13
13580        vmovdqu	OWORD PTR [rsp+296], xmm14
13581        vmovdqu	OWORD PTR [rsp+312], xmm15
13582        vpxor	xmm4, xmm4, xmm4
13583        vpxor	xmm6, xmm6, xmm6
13584        mov	edx, ebx
13585        cmp	edx, 12
13586        je	L_AES_GCM_decrypt_avx2_iv_12
13587        ; Calculate values when IV is not 12 bytes
13588        ; H = Encrypt X(=0)
13589        vmovdqu	xmm5, OWORD PTR [rsi]
13590        vaesenc	xmm5, xmm5, [rsi+16]
13591        vaesenc	xmm5, xmm5, [rsi+32]
13592        vaesenc	xmm5, xmm5, [rsi+48]
13593        vaesenc	xmm5, xmm5, [rsi+64]
13594        vaesenc	xmm5, xmm5, [rsi+80]
13595        vaesenc	xmm5, xmm5, [rsi+96]
13596        vaesenc	xmm5, xmm5, [rsi+112]
13597        vaesenc	xmm5, xmm5, [rsi+128]
13598        vaesenc	xmm5, xmm5, [rsi+144]
13599        cmp	r9d, 11
13600        vmovdqu	xmm0, OWORD PTR [rsi+160]
13601        jl	L_AES_GCM_decrypt_avx2_calc_iv_1_aesenc_avx_last
13602        vaesenc	xmm5, xmm5, xmm0
13603        vaesenc	xmm5, xmm5, [rsi+176]
13604        cmp	r9d, 13
13605        vmovdqu	xmm0, OWORD PTR [rsi+192]
13606        jl	L_AES_GCM_decrypt_avx2_calc_iv_1_aesenc_avx_last
13607        vaesenc	xmm5, xmm5, xmm0
13608        vaesenc	xmm5, xmm5, [rsi+208]
13609        vmovdqu	xmm0, OWORD PTR [rsi+224]
13610L_AES_GCM_decrypt_avx2_calc_iv_1_aesenc_avx_last:
13611        vaesenclast	xmm5, xmm5, xmm0
13612        vpshufb	xmm5, xmm5, OWORD PTR L_avx2_aes_gcm_bswap_mask
13613        ; Calc counter
13614        ; Initialization vector
13615        cmp	edx, 0
13616        mov	rcx, 0
13617        je	L_AES_GCM_decrypt_avx2_calc_iv_done
13618        cmp	edx, 16
13619        jl	L_AES_GCM_decrypt_avx2_calc_iv_lt16
13620        and	edx, 4294967280
13621L_AES_GCM_decrypt_avx2_calc_iv_16_loop:
13622        vmovdqu	xmm0, OWORD PTR [rax+rcx]
13623        vpshufb	xmm0, xmm0, OWORD PTR L_avx2_aes_gcm_bswap_mask
13624        vpxor	xmm4, xmm4, xmm0
13625        ; ghash_gfmul_avx
13626        vpclmulqdq	xmm2, xmm5, xmm4, 16
13627        vpclmulqdq	xmm1, xmm5, xmm4, 1
13628        vpclmulqdq	xmm0, xmm5, xmm4, 0
13629        vpclmulqdq	xmm3, xmm5, xmm4, 17
13630        vpxor	xmm2, xmm2, xmm1
13631        vpslldq	xmm1, xmm2, 8
13632        vpsrldq	xmm2, xmm2, 8
13633        vpxor	xmm7, xmm0, xmm1
13634        vpxor	xmm4, xmm3, xmm2
13635        ; ghash_mid
13636        vpsrld	xmm0, xmm7, 31
13637        vpsrld	xmm1, xmm4, 31
13638        vpslld	xmm7, xmm7, 1
13639        vpslld	xmm4, xmm4, 1
13640        vpsrldq	xmm2, xmm0, 12
13641        vpslldq	xmm0, xmm0, 4
13642        vpslldq	xmm1, xmm1, 4
13643        vpor	xmm4, xmm4, xmm2
13644        vpor	xmm7, xmm7, xmm0
13645        vpor	xmm4, xmm4, xmm1
13646        ; ghash_red
13647        vmovdqu	xmm2, OWORD PTR L_avx2_aes_gcm_mod2_128
13648        vpclmulqdq	xmm0, xmm7, xmm2, 16
13649        vpshufd	xmm1, xmm7, 78
13650        vpxor	xmm1, xmm1, xmm0
13651        vpclmulqdq	xmm0, xmm1, xmm2, 16
13652        vpshufd	xmm1, xmm1, 78
13653        vpxor	xmm1, xmm1, xmm0
13654        vpxor	xmm4, xmm4, xmm1
13655        add	ecx, 16
13656        cmp	ecx, edx
13657        jl	L_AES_GCM_decrypt_avx2_calc_iv_16_loop
13658        mov	edx, ebx
13659        cmp	ecx, edx
13660        je	L_AES_GCM_decrypt_avx2_calc_iv_done
13661L_AES_GCM_decrypt_avx2_calc_iv_lt16:
13662        vpxor	xmm0, xmm0, xmm0
13663        xor	ebx, ebx
13664        vmovdqu	OWORD PTR [rsp], xmm0
13665L_AES_GCM_decrypt_avx2_calc_iv_loop:
13666        movzx	r13d, BYTE PTR [rax+rcx]
13667        mov	BYTE PTR [rsp+rbx], r13b
13668        inc	ecx
13669        inc	ebx
13670        cmp	ecx, edx
13671        jl	L_AES_GCM_decrypt_avx2_calc_iv_loop
13672        vmovdqu	xmm0, OWORD PTR [rsp]
13673        vpshufb	xmm0, xmm0, OWORD PTR L_avx2_aes_gcm_bswap_mask
13674        vpxor	xmm4, xmm4, xmm0
13675        ; ghash_gfmul_avx
13676        vpclmulqdq	xmm2, xmm5, xmm4, 16
13677        vpclmulqdq	xmm1, xmm5, xmm4, 1
13678        vpclmulqdq	xmm0, xmm5, xmm4, 0
13679        vpclmulqdq	xmm3, xmm5, xmm4, 17
13680        vpxor	xmm2, xmm2, xmm1
13681        vpslldq	xmm1, xmm2, 8
13682        vpsrldq	xmm2, xmm2, 8
13683        vpxor	xmm7, xmm0, xmm1
13684        vpxor	xmm4, xmm3, xmm2
13685        ; ghash_mid
13686        vpsrld	xmm0, xmm7, 31
13687        vpsrld	xmm1, xmm4, 31
13688        vpslld	xmm7, xmm7, 1
13689        vpslld	xmm4, xmm4, 1
13690        vpsrldq	xmm2, xmm0, 12
13691        vpslldq	xmm0, xmm0, 4
13692        vpslldq	xmm1, xmm1, 4
13693        vpor	xmm4, xmm4, xmm2
13694        vpor	xmm7, xmm7, xmm0
13695        vpor	xmm4, xmm4, xmm1
13696        ; ghash_red
13697        vmovdqu	xmm2, OWORD PTR L_avx2_aes_gcm_mod2_128
13698        vpclmulqdq	xmm0, xmm7, xmm2, 16
13699        vpshufd	xmm1, xmm7, 78
13700        vpxor	xmm1, xmm1, xmm0
13701        vpclmulqdq	xmm0, xmm1, xmm2, 16
13702        vpshufd	xmm1, xmm1, 78
13703        vpxor	xmm1, xmm1, xmm0
13704        vpxor	xmm4, xmm4, xmm1
13705L_AES_GCM_decrypt_avx2_calc_iv_done:
13706        ; T = Encrypt counter
13707        vpxor	xmm0, xmm0, xmm0
13708        shl	edx, 3
13709        vmovq	xmm0, rdx
13710        vpxor	xmm4, xmm4, xmm0
13711        ; ghash_gfmul_avx
13712        vpclmulqdq	xmm2, xmm5, xmm4, 16
13713        vpclmulqdq	xmm1, xmm5, xmm4, 1
13714        vpclmulqdq	xmm0, xmm5, xmm4, 0
13715        vpclmulqdq	xmm3, xmm5, xmm4, 17
13716        vpxor	xmm2, xmm2, xmm1
13717        vpslldq	xmm1, xmm2, 8
13718        vpsrldq	xmm2, xmm2, 8
13719        vpxor	xmm7, xmm0, xmm1
13720        vpxor	xmm4, xmm3, xmm2
13721        ; ghash_mid
13722        vpsrld	xmm0, xmm7, 31
13723        vpsrld	xmm1, xmm4, 31
13724        vpslld	xmm7, xmm7, 1
13725        vpslld	xmm4, xmm4, 1
13726        vpsrldq	xmm2, xmm0, 12
13727        vpslldq	xmm0, xmm0, 4
13728        vpslldq	xmm1, xmm1, 4
13729        vpor	xmm4, xmm4, xmm2
13730        vpor	xmm7, xmm7, xmm0
13731        vpor	xmm4, xmm4, xmm1
13732        ; ghash_red
13733        vmovdqu	xmm2, OWORD PTR L_avx2_aes_gcm_mod2_128
13734        vpclmulqdq	xmm0, xmm7, xmm2, 16
13735        vpshufd	xmm1, xmm7, 78
13736        vpxor	xmm1, xmm1, xmm0
13737        vpclmulqdq	xmm0, xmm1, xmm2, 16
13738        vpshufd	xmm1, xmm1, 78
13739        vpxor	xmm1, xmm1, xmm0
13740        vpxor	xmm4, xmm4, xmm1
13741        vpshufb	xmm4, xmm4, OWORD PTR L_avx2_aes_gcm_bswap_mask
13742        ;   Encrypt counter
13743        vmovdqu	xmm15, OWORD PTR [rsi]
13744        vpxor	xmm15, xmm15, xmm4
13745        vaesenc	xmm15, xmm15, [rsi+16]
13746        vaesenc	xmm15, xmm15, [rsi+32]
13747        vaesenc	xmm15, xmm15, [rsi+48]
13748        vaesenc	xmm15, xmm15, [rsi+64]
13749        vaesenc	xmm15, xmm15, [rsi+80]
13750        vaesenc	xmm15, xmm15, [rsi+96]
13751        vaesenc	xmm15, xmm15, [rsi+112]
13752        vaesenc	xmm15, xmm15, [rsi+128]
13753        vaesenc	xmm15, xmm15, [rsi+144]
13754        cmp	r9d, 11
13755        vmovdqu	xmm0, OWORD PTR [rsi+160]
13756        jl	L_AES_GCM_decrypt_avx2_calc_iv_2_aesenc_avx_last
13757        vaesenc	xmm15, xmm15, xmm0
13758        vaesenc	xmm15, xmm15, [rsi+176]
13759        cmp	r9d, 13
13760        vmovdqu	xmm0, OWORD PTR [rsi+192]
13761        jl	L_AES_GCM_decrypt_avx2_calc_iv_2_aesenc_avx_last
13762        vaesenc	xmm15, xmm15, xmm0
13763        vaesenc	xmm15, xmm15, [rsi+208]
13764        vmovdqu	xmm0, OWORD PTR [rsi+224]
13765L_AES_GCM_decrypt_avx2_calc_iv_2_aesenc_avx_last:
13766        vaesenclast	xmm15, xmm15, xmm0
13767        jmp	L_AES_GCM_decrypt_avx2_iv_done
13768L_AES_GCM_decrypt_avx2_iv_12:
13769        ; # Calculate values when IV is 12 bytes
13770        ; Set counter based on IV
13771        vmovdqu	xmm4, OWORD PTR L_avx2_aes_gcm_bswap_one
13772        vmovdqu	xmm5, OWORD PTR [rsi]
13773        vpblendd	xmm4, xmm4, [rax], 7
13774        ; H = Encrypt X(=0) and T = Encrypt counter
13775        vmovdqu	xmm7, OWORD PTR [rsi+16]
13776        vpxor	xmm15, xmm4, xmm5
13777        vaesenc	xmm5, xmm5, xmm7
13778        vaesenc	xmm15, xmm15, xmm7
13779        vmovdqu	xmm0, OWORD PTR [rsi+32]
13780        vaesenc	xmm5, xmm5, xmm0
13781        vaesenc	xmm15, xmm15, xmm0
13782        vmovdqu	xmm0, OWORD PTR [rsi+48]
13783        vaesenc	xmm5, xmm5, xmm0
13784        vaesenc	xmm15, xmm15, xmm0
13785        vmovdqu	xmm0, OWORD PTR [rsi+64]
13786        vaesenc	xmm5, xmm5, xmm0
13787        vaesenc	xmm15, xmm15, xmm0
13788        vmovdqu	xmm0, OWORD PTR [rsi+80]
13789        vaesenc	xmm5, xmm5, xmm0
13790        vaesenc	xmm15, xmm15, xmm0
13791        vmovdqu	xmm0, OWORD PTR [rsi+96]
13792        vaesenc	xmm5, xmm5, xmm0
13793        vaesenc	xmm15, xmm15, xmm0
13794        vmovdqu	xmm0, OWORD PTR [rsi+112]
13795        vaesenc	xmm5, xmm5, xmm0
13796        vaesenc	xmm15, xmm15, xmm0
13797        vmovdqu	xmm0, OWORD PTR [rsi+128]
13798        vaesenc	xmm5, xmm5, xmm0
13799        vaesenc	xmm15, xmm15, xmm0
13800        vmovdqu	xmm0, OWORD PTR [rsi+144]
13801        vaesenc	xmm5, xmm5, xmm0
13802        vaesenc	xmm15, xmm15, xmm0
13803        cmp	r9d, 11
13804        vmovdqu	xmm0, OWORD PTR [rsi+160]
13805        jl	L_AES_GCM_decrypt_avx2_calc_iv_12_last
13806        vaesenc	xmm5, xmm5, xmm0
13807        vaesenc	xmm15, xmm15, xmm0
13808        vmovdqu	xmm0, OWORD PTR [rsi+176]
13809        vaesenc	xmm5, xmm5, xmm0
13810        vaesenc	xmm15, xmm15, xmm0
13811        cmp	r9d, 13
13812        vmovdqu	xmm0, OWORD PTR [rsi+192]
13813        jl	L_AES_GCM_decrypt_avx2_calc_iv_12_last
13814        vaesenc	xmm5, xmm5, xmm0
13815        vaesenc	xmm15, xmm15, xmm0
13816        vmovdqu	xmm0, OWORD PTR [rsi+208]
13817        vaesenc	xmm5, xmm5, xmm0
13818        vaesenc	xmm15, xmm15, xmm0
13819        vmovdqu	xmm0, OWORD PTR [rsi+224]
13820L_AES_GCM_decrypt_avx2_calc_iv_12_last:
13821        vaesenclast	xmm5, xmm5, xmm0
13822        vaesenclast	xmm15, xmm15, xmm0
13823        vpshufb	xmm5, xmm5, OWORD PTR L_avx2_aes_gcm_bswap_mask
13824L_AES_GCM_decrypt_avx2_iv_done:
13825        ; Additional authentication data
13826        mov	edx, r11d
13827        cmp	edx, 0
13828        je	L_AES_GCM_decrypt_avx2_calc_aad_done
13829        xor	ecx, ecx
13830        cmp	edx, 16
13831        jl	L_AES_GCM_decrypt_avx2_calc_aad_lt16
13832        and	edx, 4294967280
13833L_AES_GCM_decrypt_avx2_calc_aad_16_loop:
13834        vmovdqu	xmm0, OWORD PTR [r12+rcx]
13835        vpshufb	xmm0, xmm0, OWORD PTR L_avx2_aes_gcm_bswap_mask
13836        vpxor	xmm6, xmm6, xmm0
13837        ; ghash_gfmul_avx
13838        vpclmulqdq	xmm2, xmm5, xmm6, 16
13839        vpclmulqdq	xmm1, xmm5, xmm6, 1
13840        vpclmulqdq	xmm0, xmm5, xmm6, 0
13841        vpclmulqdq	xmm3, xmm5, xmm6, 17
13842        vpxor	xmm2, xmm2, xmm1
13843        vpslldq	xmm1, xmm2, 8
13844        vpsrldq	xmm2, xmm2, 8
13845        vpxor	xmm7, xmm0, xmm1
13846        vpxor	xmm6, xmm3, xmm2
13847        ; ghash_mid
13848        vpsrld	xmm0, xmm7, 31
13849        vpsrld	xmm1, xmm6, 31
13850        vpslld	xmm7, xmm7, 1
13851        vpslld	xmm6, xmm6, 1
13852        vpsrldq	xmm2, xmm0, 12
13853        vpslldq	xmm0, xmm0, 4
13854        vpslldq	xmm1, xmm1, 4
13855        vpor	xmm6, xmm6, xmm2
13856        vpor	xmm7, xmm7, xmm0
13857        vpor	xmm6, xmm6, xmm1
13858        ; ghash_red
13859        vmovdqu	xmm2, OWORD PTR L_avx2_aes_gcm_mod2_128
13860        vpclmulqdq	xmm0, xmm7, xmm2, 16
13861        vpshufd	xmm1, xmm7, 78
13862        vpxor	xmm1, xmm1, xmm0
13863        vpclmulqdq	xmm0, xmm1, xmm2, 16
13864        vpshufd	xmm1, xmm1, 78
13865        vpxor	xmm1, xmm1, xmm0
13866        vpxor	xmm6, xmm6, xmm1
13867        add	ecx, 16
13868        cmp	ecx, edx
13869        jl	L_AES_GCM_decrypt_avx2_calc_aad_16_loop
13870        mov	edx, r11d
13871        cmp	ecx, edx
13872        je	L_AES_GCM_decrypt_avx2_calc_aad_done
13873L_AES_GCM_decrypt_avx2_calc_aad_lt16:
13874        vpxor	xmm0, xmm0, xmm0
13875        xor	ebx, ebx
13876        vmovdqu	OWORD PTR [rsp], xmm0
13877L_AES_GCM_decrypt_avx2_calc_aad_loop:
13878        movzx	r13d, BYTE PTR [r12+rcx]
13879        mov	BYTE PTR [rsp+rbx], r13b
13880        inc	ecx
13881        inc	ebx
13882        cmp	ecx, edx
13883        jl	L_AES_GCM_decrypt_avx2_calc_aad_loop
13884        vmovdqu	xmm0, OWORD PTR [rsp]
13885        vpshufb	xmm0, xmm0, OWORD PTR L_avx2_aes_gcm_bswap_mask
13886        vpxor	xmm6, xmm6, xmm0
13887        ; ghash_gfmul_avx
13888        vpclmulqdq	xmm2, xmm5, xmm6, 16
13889        vpclmulqdq	xmm1, xmm5, xmm6, 1
13890        vpclmulqdq	xmm0, xmm5, xmm6, 0
13891        vpclmulqdq	xmm3, xmm5, xmm6, 17
13892        vpxor	xmm2, xmm2, xmm1
13893        vpslldq	xmm1, xmm2, 8
13894        vpsrldq	xmm2, xmm2, 8
13895        vpxor	xmm7, xmm0, xmm1
13896        vpxor	xmm6, xmm3, xmm2
13897        ; ghash_mid
13898        vpsrld	xmm0, xmm7, 31
13899        vpsrld	xmm1, xmm6, 31
13900        vpslld	xmm7, xmm7, 1
13901        vpslld	xmm6, xmm6, 1
13902        vpsrldq	xmm2, xmm0, 12
13903        vpslldq	xmm0, xmm0, 4
13904        vpslldq	xmm1, xmm1, 4
13905        vpor	xmm6, xmm6, xmm2
13906        vpor	xmm7, xmm7, xmm0
13907        vpor	xmm6, xmm6, xmm1
13908        ; ghash_red
13909        vmovdqu	xmm2, OWORD PTR L_avx2_aes_gcm_mod2_128
13910        vpclmulqdq	xmm0, xmm7, xmm2, 16
13911        vpshufd	xmm1, xmm7, 78
13912        vpxor	xmm1, xmm1, xmm0
13913        vpclmulqdq	xmm0, xmm1, xmm2, 16
13914        vpshufd	xmm1, xmm1, 78
13915        vpxor	xmm1, xmm1, xmm0
13916        vpxor	xmm6, xmm6, xmm1
13917L_AES_GCM_decrypt_avx2_calc_aad_done:
13918        ; Calculate counter and H
13919        vpsrlq	xmm1, xmm5, 63
13920        vpsllq	xmm0, xmm5, 1
13921        vpslldq	xmm1, xmm1, 8
13922        vpor	xmm0, xmm0, xmm1
13923        vpshufd	xmm5, xmm5, 255
13924        vpsrad	xmm5, xmm5, 31
13925        vpshufb	xmm4, xmm4, OWORD PTR L_avx2_aes_gcm_bswap_epi64
13926        vpand	xmm5, xmm5, OWORD PTR L_avx2_aes_gcm_mod2_128
13927        vpaddd	xmm4, xmm4, OWORD PTR L_avx2_aes_gcm_one
13928        vpxor	xmm5, xmm5, xmm0
13929        xor	ebx, ebx
13930        cmp	r10d, 128
13931        mov	r13d, r10d
13932        jl	L_AES_GCM_decrypt_avx2_done_128
13933        and	r13d, 4294967168
13934        vmovdqu	OWORD PTR [rsp+128], xmm4
13935        vmovdqu	OWORD PTR [rsp+144], xmm15
13936        vmovdqu	xmm3, OWORD PTR L_avx2_aes_gcm_mod2_128
13937        ; H ^ 1 and H ^ 2
13938        vpclmulqdq	xmm9, xmm5, xmm5, 0
13939        vpclmulqdq	xmm10, xmm5, xmm5, 17
13940        vpclmulqdq	xmm8, xmm9, xmm3, 16
13941        vpshufd	xmm9, xmm9, 78
13942        vpxor	xmm9, xmm9, xmm8
13943        vpclmulqdq	xmm8, xmm9, xmm3, 16
13944        vpshufd	xmm9, xmm9, 78
13945        vpxor	xmm9, xmm9, xmm8
13946        vpxor	xmm0, xmm10, xmm9
13947        vmovdqu	OWORD PTR [rsp], xmm5
13948        vmovdqu	OWORD PTR [rsp+16], xmm0
13949        ; H ^ 3 and H ^ 4
13950        vpclmulqdq	xmm11, xmm0, xmm5, 16
13951        vpclmulqdq	xmm10, xmm0, xmm5, 1
13952        vpclmulqdq	xmm9, xmm0, xmm5, 0
13953        vpclmulqdq	xmm12, xmm0, xmm5, 17
13954        vpclmulqdq	xmm13, xmm0, xmm0, 0
13955        vpclmulqdq	xmm14, xmm0, xmm0, 17
13956        vpxor	xmm11, xmm11, xmm10
13957        vpslldq	xmm10, xmm11, 8
13958        vpsrldq	xmm11, xmm11, 8
13959        vpxor	xmm10, xmm10, xmm9
13960        vpclmulqdq	xmm8, xmm13, xmm3, 16
13961        vpclmulqdq	xmm9, xmm10, xmm3, 16
13962        vpshufd	xmm10, xmm10, 78
13963        vpshufd	xmm13, xmm13, 78
13964        vpxor	xmm10, xmm10, xmm9
13965        vpxor	xmm13, xmm13, xmm8
13966        vpclmulqdq	xmm9, xmm10, xmm3, 16
13967        vpclmulqdq	xmm8, xmm13, xmm3, 16
13968        vpshufd	xmm10, xmm10, 78
13969        vpshufd	xmm13, xmm13, 78
13970        vpxor	xmm12, xmm12, xmm11
13971        vpxor	xmm13, xmm13, xmm8
13972        vpxor	xmm10, xmm10, xmm12
13973        vpxor	xmm2, xmm13, xmm14
13974        vpxor	xmm1, xmm10, xmm9
13975        vmovdqu	OWORD PTR [rsp+32], xmm1
13976        vmovdqu	OWORD PTR [rsp+48], xmm2
13977        ; H ^ 5 and H ^ 6
13978        vpclmulqdq	xmm11, xmm1, xmm0, 16
13979        vpclmulqdq	xmm10, xmm1, xmm0, 1
13980        vpclmulqdq	xmm9, xmm1, xmm0, 0
13981        vpclmulqdq	xmm12, xmm1, xmm0, 17
13982        vpclmulqdq	xmm13, xmm1, xmm1, 0
13983        vpclmulqdq	xmm14, xmm1, xmm1, 17
13984        vpxor	xmm11, xmm11, xmm10
13985        vpslldq	xmm10, xmm11, 8
13986        vpsrldq	xmm11, xmm11, 8
13987        vpxor	xmm10, xmm10, xmm9
13988        vpclmulqdq	xmm8, xmm13, xmm3, 16
13989        vpclmulqdq	xmm9, xmm10, xmm3, 16
13990        vpshufd	xmm10, xmm10, 78
13991        vpshufd	xmm13, xmm13, 78
13992        vpxor	xmm10, xmm10, xmm9
13993        vpxor	xmm13, xmm13, xmm8
13994        vpclmulqdq	xmm9, xmm10, xmm3, 16
13995        vpclmulqdq	xmm8, xmm13, xmm3, 16
13996        vpshufd	xmm10, xmm10, 78
13997        vpshufd	xmm13, xmm13, 78
13998        vpxor	xmm12, xmm12, xmm11
13999        vpxor	xmm13, xmm13, xmm8
14000        vpxor	xmm10, xmm10, xmm12
14001        vpxor	xmm0, xmm13, xmm14
14002        vpxor	xmm7, xmm10, xmm9
14003        vmovdqu	OWORD PTR [rsp+64], xmm7
14004        vmovdqu	OWORD PTR [rsp+80], xmm0
14005        ; H ^ 7 and H ^ 8
14006        vpclmulqdq	xmm11, xmm2, xmm1, 16
14007        vpclmulqdq	xmm10, xmm2, xmm1, 1
14008        vpclmulqdq	xmm9, xmm2, xmm1, 0
14009        vpclmulqdq	xmm12, xmm2, xmm1, 17
14010        vpclmulqdq	xmm13, xmm2, xmm2, 0
14011        vpclmulqdq	xmm14, xmm2, xmm2, 17
14012        vpxor	xmm11, xmm11, xmm10
14013        vpslldq	xmm10, xmm11, 8
14014        vpsrldq	xmm11, xmm11, 8
14015        vpxor	xmm10, xmm10, xmm9
14016        vpclmulqdq	xmm8, xmm13, xmm3, 16
14017        vpclmulqdq	xmm9, xmm10, xmm3, 16
14018        vpshufd	xmm10, xmm10, 78
14019        vpshufd	xmm13, xmm13, 78
14020        vpxor	xmm10, xmm10, xmm9
14021        vpxor	xmm13, xmm13, xmm8
14022        vpclmulqdq	xmm9, xmm10, xmm3, 16
14023        vpclmulqdq	xmm8, xmm13, xmm3, 16
14024        vpshufd	xmm10, xmm10, 78
14025        vpshufd	xmm13, xmm13, 78
14026        vpxor	xmm12, xmm12, xmm11
14027        vpxor	xmm13, xmm13, xmm8
14028        vpxor	xmm10, xmm10, xmm12
14029        vpxor	xmm0, xmm13, xmm14
14030        vpxor	xmm7, xmm10, xmm9
14031        vmovdqu	OWORD PTR [rsp+96], xmm7
14032        vmovdqu	OWORD PTR [rsp+112], xmm0
14033L_AES_GCM_decrypt_avx2_ghash_128:
14034        ; aesenc_128_ghash
14035        lea	rcx, QWORD PTR [rdi+rbx]
14036        lea	rdx, QWORD PTR [r8+rbx]
14037        ; aesenc_ctr
14038        vmovdqu	xmm0, OWORD PTR [rsp+128]
14039        vmovdqu	xmm1, OWORD PTR L_avx2_aes_gcm_bswap_epi64
14040        vpaddd	xmm9, xmm0, OWORD PTR L_avx2_aes_gcm_one
14041        vpshufb	xmm8, xmm0, xmm1
14042        vpaddd	xmm10, xmm0, OWORD PTR L_avx2_aes_gcm_two
14043        vpshufb	xmm9, xmm9, xmm1
14044        vpaddd	xmm11, xmm0, OWORD PTR L_avx2_aes_gcm_three
14045        vpshufb	xmm10, xmm10, xmm1
14046        vpaddd	xmm12, xmm0, OWORD PTR L_avx2_aes_gcm_four
14047        vpshufb	xmm11, xmm11, xmm1
14048        vpaddd	xmm13, xmm0, OWORD PTR L_avx2_aes_gcm_five
14049        vpshufb	xmm12, xmm12, xmm1
14050        vpaddd	xmm14, xmm0, OWORD PTR L_avx2_aes_gcm_six
14051        vpshufb	xmm13, xmm13, xmm1
14052        vpaddd	xmm15, xmm0, OWORD PTR L_avx2_aes_gcm_seven
14053        vpshufb	xmm14, xmm14, xmm1
14054        vpaddd	xmm0, xmm0, OWORD PTR L_avx2_aes_gcm_eight
14055        vpshufb	xmm15, xmm15, xmm1
14056        ; aesenc_xor
14057        vmovdqu	xmm7, OWORD PTR [rsi]
14058        vmovdqu	OWORD PTR [rsp+128], xmm0
14059        vpxor	xmm8, xmm8, xmm7
14060        vpxor	xmm9, xmm9, xmm7
14061        vpxor	xmm10, xmm10, xmm7
14062        vpxor	xmm11, xmm11, xmm7
14063        vpxor	xmm12, xmm12, xmm7
14064        vpxor	xmm13, xmm13, xmm7
14065        vpxor	xmm14, xmm14, xmm7
14066        vpxor	xmm15, xmm15, xmm7
14067        ; aesenc_pclmul_1
14068        vmovdqu	xmm1, OWORD PTR [rcx]
14069        vmovdqu	xmm0, OWORD PTR [rsi+16]
14070        vpshufb	xmm1, xmm1, OWORD PTR L_avx2_aes_gcm_bswap_mask
14071        vmovdqu	xmm2, OWORD PTR [rsp+112]
14072        vpxor	xmm1, xmm1, xmm6
14073        vpclmulqdq	xmm5, xmm1, xmm2, 16
14074        vpclmulqdq	xmm3, xmm1, xmm2, 1
14075        vpclmulqdq	xmm6, xmm1, xmm2, 0
14076        vpclmulqdq	xmm7, xmm1, xmm2, 17
14077        vaesenc	xmm8, xmm8, xmm0
14078        vaesenc	xmm9, xmm9, xmm0
14079        vaesenc	xmm10, xmm10, xmm0
14080        vaesenc	xmm11, xmm11, xmm0
14081        vaesenc	xmm12, xmm12, xmm0
14082        vaesenc	xmm13, xmm13, xmm0
14083        vaesenc	xmm14, xmm14, xmm0
14084        vaesenc	xmm15, xmm15, xmm0
14085        ; aesenc_pclmul_2
14086        vmovdqu	xmm1, OWORD PTR [rcx+16]
14087        vmovdqu	xmm0, OWORD PTR [rsp+96]
14088        vpshufb	xmm1, xmm1, OWORD PTR L_avx2_aes_gcm_bswap_mask
14089        vpxor	xmm5, xmm5, xmm3
14090        vpclmulqdq	xmm2, xmm1, xmm0, 16
14091        vpclmulqdq	xmm3, xmm1, xmm0, 1
14092        vpclmulqdq	xmm4, xmm1, xmm0, 0
14093        vpclmulqdq	xmm1, xmm1, xmm0, 17
14094        vmovdqu	xmm0, OWORD PTR [rsi+32]
14095        vpxor	xmm7, xmm7, xmm1
14096        vaesenc	xmm8, xmm8, xmm0
14097        vaesenc	xmm9, xmm9, xmm0
14098        vaesenc	xmm10, xmm10, xmm0
14099        vaesenc	xmm11, xmm11, xmm0
14100        vaesenc	xmm12, xmm12, xmm0
14101        vaesenc	xmm13, xmm13, xmm0
14102        vaesenc	xmm14, xmm14, xmm0
14103        vaesenc	xmm15, xmm15, xmm0
14104        ; aesenc_pclmul_n
14105        vmovdqu	xmm1, OWORD PTR [rcx+32]
14106        vmovdqu	xmm0, OWORD PTR [rsp+80]
14107        vpshufb	xmm1, xmm1, OWORD PTR L_avx2_aes_gcm_bswap_mask
14108        vpxor	xmm5, xmm5, xmm2
14109        vpclmulqdq	xmm2, xmm1, xmm0, 16
14110        vpxor	xmm5, xmm5, xmm3
14111        vpclmulqdq	xmm3, xmm1, xmm0, 1
14112        vpxor	xmm6, xmm6, xmm4
14113        vpclmulqdq	xmm4, xmm1, xmm0, 0
14114        vpclmulqdq	xmm1, xmm1, xmm0, 17
14115        vmovdqu	xmm0, OWORD PTR [rsi+48]
14116        vpxor	xmm7, xmm7, xmm1
14117        vaesenc	xmm8, xmm8, xmm0
14118        vaesenc	xmm9, xmm9, xmm0
14119        vaesenc	xmm10, xmm10, xmm0
14120        vaesenc	xmm11, xmm11, xmm0
14121        vaesenc	xmm12, xmm12, xmm0
14122        vaesenc	xmm13, xmm13, xmm0
14123        vaesenc	xmm14, xmm14, xmm0
14124        vaesenc	xmm15, xmm15, xmm0
14125        ; aesenc_pclmul_n
14126        vmovdqu	xmm1, OWORD PTR [rcx+48]
14127        vmovdqu	xmm0, OWORD PTR [rsp+64]
14128        vpshufb	xmm1, xmm1, OWORD PTR L_avx2_aes_gcm_bswap_mask
14129        vpxor	xmm5, xmm5, xmm2
14130        vpclmulqdq	xmm2, xmm1, xmm0, 16
14131        vpxor	xmm5, xmm5, xmm3
14132        vpclmulqdq	xmm3, xmm1, xmm0, 1
14133        vpxor	xmm6, xmm6, xmm4
14134        vpclmulqdq	xmm4, xmm1, xmm0, 0
14135        vpclmulqdq	xmm1, xmm1, xmm0, 17
14136        vmovdqu	xmm0, OWORD PTR [rsi+64]
14137        vpxor	xmm7, xmm7, xmm1
14138        vaesenc	xmm8, xmm8, xmm0
14139        vaesenc	xmm9, xmm9, xmm0
14140        vaesenc	xmm10, xmm10, xmm0
14141        vaesenc	xmm11, xmm11, xmm0
14142        vaesenc	xmm12, xmm12, xmm0
14143        vaesenc	xmm13, xmm13, xmm0
14144        vaesenc	xmm14, xmm14, xmm0
14145        vaesenc	xmm15, xmm15, xmm0
14146        ; aesenc_pclmul_n
14147        vmovdqu	xmm1, OWORD PTR [rcx+64]
14148        vmovdqu	xmm0, OWORD PTR [rsp+48]
14149        vpshufb	xmm1, xmm1, OWORD PTR L_avx2_aes_gcm_bswap_mask
14150        vpxor	xmm5, xmm5, xmm2
14151        vpclmulqdq	xmm2, xmm1, xmm0, 16
14152        vpxor	xmm5, xmm5, xmm3
14153        vpclmulqdq	xmm3, xmm1, xmm0, 1
14154        vpxor	xmm6, xmm6, xmm4
14155        vpclmulqdq	xmm4, xmm1, xmm0, 0
14156        vpclmulqdq	xmm1, xmm1, xmm0, 17
14157        vmovdqu	xmm0, OWORD PTR [rsi+80]
14158        vpxor	xmm7, xmm7, xmm1
14159        vaesenc	xmm8, xmm8, xmm0
14160        vaesenc	xmm9, xmm9, xmm0
14161        vaesenc	xmm10, xmm10, xmm0
14162        vaesenc	xmm11, xmm11, xmm0
14163        vaesenc	xmm12, xmm12, xmm0
14164        vaesenc	xmm13, xmm13, xmm0
14165        vaesenc	xmm14, xmm14, xmm0
14166        vaesenc	xmm15, xmm15, xmm0
14167        ; aesenc_pclmul_n
14168        vmovdqu	xmm1, OWORD PTR [rcx+80]
14169        vmovdqu	xmm0, OWORD PTR [rsp+32]
14170        vpshufb	xmm1, xmm1, OWORD PTR L_avx2_aes_gcm_bswap_mask
14171        vpxor	xmm5, xmm5, xmm2
14172        vpclmulqdq	xmm2, xmm1, xmm0, 16
14173        vpxor	xmm5, xmm5, xmm3
14174        vpclmulqdq	xmm3, xmm1, xmm0, 1
14175        vpxor	xmm6, xmm6, xmm4
14176        vpclmulqdq	xmm4, xmm1, xmm0, 0
14177        vpclmulqdq	xmm1, xmm1, xmm0, 17
14178        vmovdqu	xmm0, OWORD PTR [rsi+96]
14179        vpxor	xmm7, xmm7, xmm1
14180        vaesenc	xmm8, xmm8, xmm0
14181        vaesenc	xmm9, xmm9, xmm0
14182        vaesenc	xmm10, xmm10, xmm0
14183        vaesenc	xmm11, xmm11, xmm0
14184        vaesenc	xmm12, xmm12, xmm0
14185        vaesenc	xmm13, xmm13, xmm0
14186        vaesenc	xmm14, xmm14, xmm0
14187        vaesenc	xmm15, xmm15, xmm0
14188        ; aesenc_pclmul_n
14189        vmovdqu	xmm1, OWORD PTR [rcx+96]
14190        vmovdqu	xmm0, OWORD PTR [rsp+16]
14191        vpshufb	xmm1, xmm1, OWORD PTR L_avx2_aes_gcm_bswap_mask
14192        vpxor	xmm5, xmm5, xmm2
14193        vpclmulqdq	xmm2, xmm1, xmm0, 16
14194        vpxor	xmm5, xmm5, xmm3
14195        vpclmulqdq	xmm3, xmm1, xmm0, 1
14196        vpxor	xmm6, xmm6, xmm4
14197        vpclmulqdq	xmm4, xmm1, xmm0, 0
14198        vpclmulqdq	xmm1, xmm1, xmm0, 17
14199        vmovdqu	xmm0, OWORD PTR [rsi+112]
14200        vpxor	xmm7, xmm7, xmm1
14201        vaesenc	xmm8, xmm8, xmm0
14202        vaesenc	xmm9, xmm9, xmm0
14203        vaesenc	xmm10, xmm10, xmm0
14204        vaesenc	xmm11, xmm11, xmm0
14205        vaesenc	xmm12, xmm12, xmm0
14206        vaesenc	xmm13, xmm13, xmm0
14207        vaesenc	xmm14, xmm14, xmm0
14208        vaesenc	xmm15, xmm15, xmm0
14209        ; aesenc_pclmul_n
14210        vmovdqu	xmm1, OWORD PTR [rcx+112]
14211        vmovdqu	xmm0, OWORD PTR [rsp]
14212        vpshufb	xmm1, xmm1, OWORD PTR L_avx2_aes_gcm_bswap_mask
14213        vpxor	xmm5, xmm5, xmm2
14214        vpclmulqdq	xmm2, xmm1, xmm0, 16
14215        vpxor	xmm5, xmm5, xmm3
14216        vpclmulqdq	xmm3, xmm1, xmm0, 1
14217        vpxor	xmm6, xmm6, xmm4
14218        vpclmulqdq	xmm4, xmm1, xmm0, 0
14219        vpclmulqdq	xmm1, xmm1, xmm0, 17
14220        vmovdqu	xmm0, OWORD PTR [rsi+128]
14221        vpxor	xmm7, xmm7, xmm1
14222        vaesenc	xmm8, xmm8, xmm0
14223        vaesenc	xmm9, xmm9, xmm0
14224        vaesenc	xmm10, xmm10, xmm0
14225        vaesenc	xmm11, xmm11, xmm0
14226        vaesenc	xmm12, xmm12, xmm0
14227        vaesenc	xmm13, xmm13, xmm0
14228        vaesenc	xmm14, xmm14, xmm0
14229        vaesenc	xmm15, xmm15, xmm0
14230        ; aesenc_pclmul_l
14231        vpxor	xmm5, xmm5, xmm2
14232        vpxor	xmm6, xmm6, xmm4
14233        vpxor	xmm5, xmm5, xmm3
14234        vpslldq	xmm1, xmm5, 8
14235        vpsrldq	xmm5, xmm5, 8
14236        vmovdqu	xmm4, OWORD PTR [rsi+144]
14237        vmovdqu	xmm0, OWORD PTR L_avx2_aes_gcm_mod2_128
14238        vaesenc	xmm8, xmm8, xmm4
14239        vpxor	xmm6, xmm6, xmm1
14240        vpxor	xmm7, xmm7, xmm5
14241        vpclmulqdq	xmm3, xmm6, xmm0, 16
14242        vaesenc	xmm9, xmm9, xmm4
14243        vaesenc	xmm10, xmm10, xmm4
14244        vaesenc	xmm11, xmm11, xmm4
14245        vpshufd	xmm6, xmm6, 78
14246        vpxor	xmm6, xmm6, xmm3
14247        vpclmulqdq	xmm3, xmm6, xmm0, 16
14248        vaesenc	xmm12, xmm12, xmm4
14249        vaesenc	xmm13, xmm13, xmm4
14250        vaesenc	xmm14, xmm14, xmm4
14251        vpshufd	xmm6, xmm6, 78
14252        vpxor	xmm6, xmm6, xmm3
14253        vpxor	xmm6, xmm6, xmm7
14254        vaesenc	xmm15, xmm15, xmm4
14255        cmp	r9d, 11
14256        vmovdqu	xmm7, OWORD PTR [rsi+160]
14257        jl	L_AES_GCM_decrypt_avx2_aesenc_128_ghash_avx_done
14258        vaesenc	xmm8, xmm8, xmm7
14259        vaesenc	xmm9, xmm9, xmm7
14260        vaesenc	xmm10, xmm10, xmm7
14261        vaesenc	xmm11, xmm11, xmm7
14262        vaesenc	xmm12, xmm12, xmm7
14263        vaesenc	xmm13, xmm13, xmm7
14264        vaesenc	xmm14, xmm14, xmm7
14265        vaesenc	xmm15, xmm15, xmm7
14266        vmovdqu	xmm7, OWORD PTR [rsi+176]
14267        vaesenc	xmm8, xmm8, xmm7
14268        vaesenc	xmm9, xmm9, xmm7
14269        vaesenc	xmm10, xmm10, xmm7
14270        vaesenc	xmm11, xmm11, xmm7
14271        vaesenc	xmm12, xmm12, xmm7
14272        vaesenc	xmm13, xmm13, xmm7
14273        vaesenc	xmm14, xmm14, xmm7
14274        vaesenc	xmm15, xmm15, xmm7
14275        cmp	r9d, 13
14276        vmovdqu	xmm7, OWORD PTR [rsi+192]
14277        jl	L_AES_GCM_decrypt_avx2_aesenc_128_ghash_avx_done
14278        vaesenc	xmm8, xmm8, xmm7
14279        vaesenc	xmm9, xmm9, xmm7
14280        vaesenc	xmm10, xmm10, xmm7
14281        vaesenc	xmm11, xmm11, xmm7
14282        vaesenc	xmm12, xmm12, xmm7
14283        vaesenc	xmm13, xmm13, xmm7
14284        vaesenc	xmm14, xmm14, xmm7
14285        vaesenc	xmm15, xmm15, xmm7
14286        vmovdqu	xmm7, OWORD PTR [rsi+208]
14287        vaesenc	xmm8, xmm8, xmm7
14288        vaesenc	xmm9, xmm9, xmm7
14289        vaesenc	xmm10, xmm10, xmm7
14290        vaesenc	xmm11, xmm11, xmm7
14291        vaesenc	xmm12, xmm12, xmm7
14292        vaesenc	xmm13, xmm13, xmm7
14293        vaesenc	xmm14, xmm14, xmm7
14294        vaesenc	xmm15, xmm15, xmm7
14295        vmovdqu	xmm7, OWORD PTR [rsi+224]
14296L_AES_GCM_decrypt_avx2_aesenc_128_ghash_avx_done:
14297        ; aesenc_last
14298        vaesenclast	xmm8, xmm8, xmm7
14299        vaesenclast	xmm9, xmm9, xmm7
14300        vaesenclast	xmm10, xmm10, xmm7
14301        vaesenclast	xmm11, xmm11, xmm7
14302        vmovdqu	xmm0, OWORD PTR [rcx]
14303        vmovdqu	xmm1, OWORD PTR [rcx+16]
14304        vmovdqu	xmm2, OWORD PTR [rcx+32]
14305        vmovdqu	xmm3, OWORD PTR [rcx+48]
14306        vpxor	xmm8, xmm8, xmm0
14307        vpxor	xmm9, xmm9, xmm1
14308        vpxor	xmm10, xmm10, xmm2
14309        vpxor	xmm11, xmm11, xmm3
14310        vmovdqu	OWORD PTR [rdx], xmm8
14311        vmovdqu	OWORD PTR [rdx+16], xmm9
14312        vmovdqu	OWORD PTR [rdx+32], xmm10
14313        vmovdqu	OWORD PTR [rdx+48], xmm11
14314        vaesenclast	xmm12, xmm12, xmm7
14315        vaesenclast	xmm13, xmm13, xmm7
14316        vaesenclast	xmm14, xmm14, xmm7
14317        vaesenclast	xmm15, xmm15, xmm7
14318        vmovdqu	xmm0, OWORD PTR [rcx+64]
14319        vmovdqu	xmm1, OWORD PTR [rcx+80]
14320        vmovdqu	xmm2, OWORD PTR [rcx+96]
14321        vmovdqu	xmm3, OWORD PTR [rcx+112]
14322        vpxor	xmm12, xmm12, xmm0
14323        vpxor	xmm13, xmm13, xmm1
14324        vpxor	xmm14, xmm14, xmm2
14325        vpxor	xmm15, xmm15, xmm3
14326        vmovdqu	OWORD PTR [rdx+64], xmm12
14327        vmovdqu	OWORD PTR [rdx+80], xmm13
14328        vmovdqu	OWORD PTR [rdx+96], xmm14
14329        vmovdqu	OWORD PTR [rdx+112], xmm15
14330        ; aesenc_128_ghash - end
14331        add	ebx, 128
14332        cmp	ebx, r13d
14333        jl	L_AES_GCM_decrypt_avx2_ghash_128
14334        vmovdqu	xmm5, OWORD PTR [rsp]
14335        vmovdqu	xmm4, OWORD PTR [rsp+128]
14336        vmovdqu	xmm15, OWORD PTR [rsp+144]
14337L_AES_GCM_decrypt_avx2_done_128:
14338        cmp	ebx, r10d
14339        jge	L_AES_GCM_decrypt_avx2_done_dec
14340        mov	r13d, r10d
14341        and	r13d, 4294967280
14342        cmp	ebx, r13d
14343        jge	L_AES_GCM_decrypt_avx2_last_block_done
14344L_AES_GCM_decrypt_avx2_last_block_start:
14345        vmovdqu	xmm11, OWORD PTR [rdi+rbx]
14346        vpshufb	xmm10, xmm4, OWORD PTR L_avx2_aes_gcm_bswap_epi64
14347        vpshufb	xmm12, xmm11, OWORD PTR L_avx2_aes_gcm_bswap_mask
14348        vpaddd	xmm4, xmm4, OWORD PTR L_avx2_aes_gcm_one
14349        vpxor	xmm12, xmm12, xmm6
14350        ; aesenc_gfmul_sb
14351        vpclmulqdq	xmm2, xmm12, xmm5, 1
14352        vpclmulqdq	xmm3, xmm12, xmm5, 16
14353        vpclmulqdq	xmm1, xmm12, xmm5, 0
14354        vpclmulqdq	xmm8, xmm12, xmm5, 17
14355        vpxor	xmm10, xmm10, [rsi]
14356        vaesenc	xmm10, xmm10, [rsi+16]
14357        vpxor	xmm3, xmm3, xmm2
14358        vpslldq	xmm2, xmm3, 8
14359        vpsrldq	xmm3, xmm3, 8
14360        vaesenc	xmm10, xmm10, [rsi+32]
14361        vpxor	xmm2, xmm2, xmm1
14362        vpclmulqdq	xmm1, xmm2, OWORD PTR L_avx2_aes_gcm_mod2_128, 16
14363        vaesenc	xmm10, xmm10, [rsi+48]
14364        vaesenc	xmm10, xmm10, [rsi+64]
14365        vaesenc	xmm10, xmm10, [rsi+80]
14366        vpshufd	xmm2, xmm2, 78
14367        vpxor	xmm2, xmm2, xmm1
14368        vpclmulqdq	xmm1, xmm2, OWORD PTR L_avx2_aes_gcm_mod2_128, 16
14369        vaesenc	xmm10, xmm10, [rsi+96]
14370        vaesenc	xmm10, xmm10, [rsi+112]
14371        vaesenc	xmm10, xmm10, [rsi+128]
14372        vpshufd	xmm2, xmm2, 78
14373        vaesenc	xmm10, xmm10, [rsi+144]
14374        vpxor	xmm8, xmm8, xmm3
14375        vpxor	xmm2, xmm2, xmm8
14376        vmovdqu	xmm0, OWORD PTR [rsi+160]
14377        cmp	r9d, 11
14378        jl	L_AES_GCM_decrypt_avx2_aesenc_gfmul_sb_last
14379        vaesenc	xmm10, xmm10, xmm0
14380        vaesenc	xmm10, xmm10, [rsi+176]
14381        vmovdqu	xmm0, OWORD PTR [rsi+192]
14382        cmp	r9d, 13
14383        jl	L_AES_GCM_decrypt_avx2_aesenc_gfmul_sb_last
14384        vaesenc	xmm10, xmm10, xmm0
14385        vaesenc	xmm10, xmm10, [rsi+208]
14386        vmovdqu	xmm0, OWORD PTR [rsi+224]
14387L_AES_GCM_decrypt_avx2_aesenc_gfmul_sb_last:
14388        vaesenclast	xmm10, xmm10, xmm0
14389        vpxor	xmm6, xmm2, xmm1
14390        vpxor	xmm10, xmm10, xmm11
14391        vmovdqu	OWORD PTR [r8+rbx], xmm10
14392        add	ebx, 16
14393        cmp	ebx, r13d
14394        jl	L_AES_GCM_decrypt_avx2_last_block_start
14395L_AES_GCM_decrypt_avx2_last_block_done:
14396        mov	ecx, r10d
14397        mov	edx, r10d
14398        and	ecx, 15
14399        jz	L_AES_GCM_decrypt_avx2_done_dec
14400        ; aesenc_last15_dec
14401        vpshufb	xmm4, xmm4, OWORD PTR L_avx2_aes_gcm_bswap_epi64
14402        vpxor	xmm4, xmm4, [rsi]
14403        vaesenc	xmm4, xmm4, [rsi+16]
14404        vaesenc	xmm4, xmm4, [rsi+32]
14405        vaesenc	xmm4, xmm4, [rsi+48]
14406        vaesenc	xmm4, xmm4, [rsi+64]
14407        vaesenc	xmm4, xmm4, [rsi+80]
14408        vaesenc	xmm4, xmm4, [rsi+96]
14409        vaesenc	xmm4, xmm4, [rsi+112]
14410        vaesenc	xmm4, xmm4, [rsi+128]
14411        vaesenc	xmm4, xmm4, [rsi+144]
14412        cmp	r9d, 11
14413        vmovdqu	xmm1, OWORD PTR [rsi+160]
14414        jl	L_AES_GCM_decrypt_avx2_aesenc_last15_dec_avx_aesenc_avx_last
14415        vaesenc	xmm4, xmm4, xmm1
14416        vaesenc	xmm4, xmm4, [rsi+176]
14417        cmp	r9d, 13
14418        vmovdqu	xmm1, OWORD PTR [rsi+192]
14419        jl	L_AES_GCM_decrypt_avx2_aesenc_last15_dec_avx_aesenc_avx_last
14420        vaesenc	xmm4, xmm4, xmm1
14421        vaesenc	xmm4, xmm4, [rsi+208]
14422        vmovdqu	xmm1, OWORD PTR [rsi+224]
14423L_AES_GCM_decrypt_avx2_aesenc_last15_dec_avx_aesenc_avx_last:
14424        vaesenclast	xmm4, xmm4, xmm1
14425        xor	ecx, ecx
14426        vpxor	xmm0, xmm0, xmm0
14427        vmovdqu	OWORD PTR [rsp], xmm4
14428        vmovdqu	OWORD PTR [rsp+16], xmm0
14429L_AES_GCM_decrypt_avx2_aesenc_last15_dec_avx_loop:
14430        movzx	r13d, BYTE PTR [rdi+rbx]
14431        mov	BYTE PTR [rsp+rcx+16], r13b
14432        xor	r13b, BYTE PTR [rsp+rcx]
14433        mov	BYTE PTR [r8+rbx], r13b
14434        inc	ebx
14435        inc	ecx
14436        cmp	ebx, edx
14437        jl	L_AES_GCM_decrypt_avx2_aesenc_last15_dec_avx_loop
14438        vmovdqu	xmm4, OWORD PTR [rsp+16]
14439        vpshufb	xmm4, xmm4, OWORD PTR L_avx2_aes_gcm_bswap_mask
14440        vpxor	xmm6, xmm6, xmm4
14441        ; ghash_gfmul_red
14442        vpclmulqdq	xmm2, xmm6, xmm5, 16
14443        vpclmulqdq	xmm1, xmm6, xmm5, 1
14444        vpclmulqdq	xmm0, xmm6, xmm5, 0
14445        vpxor	xmm2, xmm2, xmm1
14446        vpslldq	xmm1, xmm2, 8
14447        vpsrldq	xmm2, xmm2, 8
14448        vpxor	xmm1, xmm1, xmm0
14449        vpclmulqdq	xmm6, xmm6, xmm5, 17
14450        vpclmulqdq	xmm0, xmm1, OWORD PTR L_avx2_aes_gcm_mod2_128, 16
14451        vpshufd	xmm1, xmm1, 78
14452        vpxor	xmm1, xmm1, xmm0
14453        vpclmulqdq	xmm0, xmm1, OWORD PTR L_avx2_aes_gcm_mod2_128, 16
14454        vpshufd	xmm1, xmm1, 78
14455        vpxor	xmm6, xmm6, xmm2
14456        vpxor	xmm6, xmm6, xmm1
14457        vpxor	xmm6, xmm6, xmm0
14458L_AES_GCM_decrypt_avx2_done_dec:
14459        ; calc_tag
14460        shl	r10, 3
14461        shl	r11, 3
14462        vmovq	xmm0, r10
14463        vmovq	xmm1, r11
14464        vpunpcklqdq	xmm0, xmm0, xmm1
14465        vpxor	xmm0, xmm0, xmm6
14466        ; ghash_gfmul_red
14467        vpclmulqdq	xmm4, xmm0, xmm5, 16
14468        vpclmulqdq	xmm3, xmm0, xmm5, 1
14469        vpclmulqdq	xmm2, xmm0, xmm5, 0
14470        vpxor	xmm4, xmm4, xmm3
14471        vpslldq	xmm3, xmm4, 8
14472        vpsrldq	xmm4, xmm4, 8
14473        vpxor	xmm3, xmm3, xmm2
14474        vpclmulqdq	xmm0, xmm0, xmm5, 17
14475        vpclmulqdq	xmm2, xmm3, OWORD PTR L_avx2_aes_gcm_mod2_128, 16
14476        vpshufd	xmm3, xmm3, 78
14477        vpxor	xmm3, xmm3, xmm2
14478        vpclmulqdq	xmm2, xmm3, OWORD PTR L_avx2_aes_gcm_mod2_128, 16
14479        vpshufd	xmm3, xmm3, 78
14480        vpxor	xmm0, xmm0, xmm4
14481        vpxor	xmm0, xmm0, xmm3
14482        vpxor	xmm0, xmm0, xmm2
14483        vpshufb	xmm0, xmm0, OWORD PTR L_avx2_aes_gcm_bswap_mask
14484        vpxor	xmm0, xmm0, xmm15
14485        ; cmp_tag
14486        cmp	r15d, 16
14487        je	L_AES_GCM_decrypt_avx2_cmp_tag_16
14488        xor	rdx, rdx
14489        xor	rax, rax
14490        vmovdqu	OWORD PTR [rsp], xmm0
14491L_AES_GCM_decrypt_avx2_cmp_tag_loop:
14492        movzx	r13d, BYTE PTR [rsp+rdx]
14493        xor	r13b, BYTE PTR [r14+rdx]
14494        or	al, r13b
14495        inc	edx
14496        cmp	edx, r15d
14497        jne	L_AES_GCM_decrypt_avx2_cmp_tag_loop
14498        cmp	al, 0
14499        sete	al
14500        jmp	L_AES_GCM_decrypt_avx2_cmp_tag_done
14501L_AES_GCM_decrypt_avx2_cmp_tag_16:
14502        vmovdqu	xmm1, OWORD PTR [r14]
14503        vpcmpeqb	xmm0, xmm0, xmm1
14504        vpmovmskb	rdx, xmm0
14505        ; %%edx == 0xFFFF then return 1 else => return 0
14506        xor	eax, eax
14507        cmp	edx, 65535
14508        sete	al
14509L_AES_GCM_decrypt_avx2_cmp_tag_done:
14510        mov	DWORD PTR [rbp], eax
14511        vzeroupper
14512        vmovdqu	xmm6, OWORD PTR [rsp+168]
14513        vmovdqu	xmm7, OWORD PTR [rsp+184]
14514        vmovdqu	xmm8, OWORD PTR [rsp+200]
14515        vmovdqu	xmm9, OWORD PTR [rsp+216]
14516        vmovdqu	xmm10, OWORD PTR [rsp+232]
14517        vmovdqu	xmm11, OWORD PTR [rsp+248]
14518        vmovdqu	xmm12, OWORD PTR [rsp+264]
14519        vmovdqu	xmm13, OWORD PTR [rsp+280]
14520        vmovdqu	xmm14, OWORD PTR [rsp+296]
14521        vmovdqu	xmm15, OWORD PTR [rsp+312]
14522        add	rsp, 328
14523        pop	rbp
14524        pop	rsi
14525        pop	r15
14526        pop	rbx
14527        pop	r14
14528        pop	r12
14529        pop	rdi
14530        pop	r13
14531        ret
14532AES_GCM_decrypt_avx2 ENDP
14533_TEXT ENDS
14534_TEXT SEGMENT READONLY PARA
14535AES_GCM_init_avx2 PROC
14536        push	rbx
14537        push	rdi
14538        push	rsi
14539        push	r12
14540        mov	rdi, rcx
14541        mov	rsi, rdx
14542        mov	r10, r8
14543        mov	r11d, r9d
14544        mov	rax, QWORD PTR [rsp+72]
14545        mov	r8, QWORD PTR [rsp+80]
14546        mov	r9, QWORD PTR [rsp+88]
14547        sub	rsp, 48
14548        vmovdqu	OWORD PTR [rsp+16], xmm6
14549        vmovdqu	OWORD PTR [rsp+32], xmm7
14550        vpxor	xmm4, xmm4, xmm4
14551        mov	edx, r11d
14552        cmp	edx, 12
14553        je	L_AES_GCM_init_avx2_iv_12
14554        ; Calculate values when IV is not 12 bytes
14555        ; H = Encrypt X(=0)
14556        vmovdqu	xmm5, OWORD PTR [rdi]
14557        vaesenc	xmm5, xmm5, [rdi+16]
14558        vaesenc	xmm5, xmm5, [rdi+32]
14559        vaesenc	xmm5, xmm5, [rdi+48]
14560        vaesenc	xmm5, xmm5, [rdi+64]
14561        vaesenc	xmm5, xmm5, [rdi+80]
14562        vaesenc	xmm5, xmm5, [rdi+96]
14563        vaesenc	xmm5, xmm5, [rdi+112]
14564        vaesenc	xmm5, xmm5, [rdi+128]
14565        vaesenc	xmm5, xmm5, [rdi+144]
14566        cmp	esi, 11
14567        vmovdqu	xmm0, OWORD PTR [rdi+160]
14568        jl	L_AES_GCM_init_avx2_calc_iv_1_aesenc_avx_last
14569        vaesenc	xmm5, xmm5, xmm0
14570        vaesenc	xmm5, xmm5, [rdi+176]
14571        cmp	esi, 13
14572        vmovdqu	xmm0, OWORD PTR [rdi+192]
14573        jl	L_AES_GCM_init_avx2_calc_iv_1_aesenc_avx_last
14574        vaesenc	xmm5, xmm5, xmm0
14575        vaesenc	xmm5, xmm5, [rdi+208]
14576        vmovdqu	xmm0, OWORD PTR [rdi+224]
14577L_AES_GCM_init_avx2_calc_iv_1_aesenc_avx_last:
14578        vaesenclast	xmm5, xmm5, xmm0
14579        vpshufb	xmm5, xmm5, OWORD PTR L_avx2_aes_gcm_bswap_mask
14580        ; Calc counter
14581        ; Initialization vector
14582        cmp	edx, 0
14583        mov	rcx, 0
14584        je	L_AES_GCM_init_avx2_calc_iv_done
14585        cmp	edx, 16
14586        jl	L_AES_GCM_init_avx2_calc_iv_lt16
14587        and	edx, 4294967280
14588L_AES_GCM_init_avx2_calc_iv_16_loop:
14589        vmovdqu	xmm0, OWORD PTR [r10+rcx]
14590        vpshufb	xmm0, xmm0, OWORD PTR L_avx2_aes_gcm_bswap_mask
14591        vpxor	xmm4, xmm4, xmm0
14592        ; ghash_gfmul_avx
14593        vpclmulqdq	xmm2, xmm5, xmm4, 16
14594        vpclmulqdq	xmm1, xmm5, xmm4, 1
14595        vpclmulqdq	xmm0, xmm5, xmm4, 0
14596        vpclmulqdq	xmm3, xmm5, xmm4, 17
14597        vpxor	xmm2, xmm2, xmm1
14598        vpslldq	xmm1, xmm2, 8
14599        vpsrldq	xmm2, xmm2, 8
14600        vpxor	xmm6, xmm0, xmm1
14601        vpxor	xmm4, xmm3, xmm2
14602        ; ghash_mid
14603        vpsrld	xmm0, xmm6, 31
14604        vpsrld	xmm1, xmm4, 31
14605        vpslld	xmm6, xmm6, 1
14606        vpslld	xmm4, xmm4, 1
14607        vpsrldq	xmm2, xmm0, 12
14608        vpslldq	xmm0, xmm0, 4
14609        vpslldq	xmm1, xmm1, 4
14610        vpor	xmm4, xmm4, xmm2
14611        vpor	xmm6, xmm6, xmm0
14612        vpor	xmm4, xmm4, xmm1
14613        ; ghash_red
14614        vmovdqu	xmm2, OWORD PTR L_avx2_aes_gcm_mod2_128
14615        vpclmulqdq	xmm0, xmm6, xmm2, 16
14616        vpshufd	xmm1, xmm6, 78
14617        vpxor	xmm1, xmm1, xmm0
14618        vpclmulqdq	xmm0, xmm1, xmm2, 16
14619        vpshufd	xmm1, xmm1, 78
14620        vpxor	xmm1, xmm1, xmm0
14621        vpxor	xmm4, xmm4, xmm1
14622        add	ecx, 16
14623        cmp	ecx, edx
14624        jl	L_AES_GCM_init_avx2_calc_iv_16_loop
14625        mov	edx, r11d
14626        cmp	ecx, edx
14627        je	L_AES_GCM_init_avx2_calc_iv_done
14628L_AES_GCM_init_avx2_calc_iv_lt16:
14629        vpxor	xmm0, xmm0, xmm0
14630        xor	ebx, ebx
14631        vmovdqu	OWORD PTR [rsp], xmm0
14632L_AES_GCM_init_avx2_calc_iv_loop:
14633        movzx	r12d, BYTE PTR [r10+rcx]
14634        mov	BYTE PTR [rsp+rbx], r12b
14635        inc	ecx
14636        inc	ebx
14637        cmp	ecx, edx
14638        jl	L_AES_GCM_init_avx2_calc_iv_loop
14639        vmovdqu	xmm0, OWORD PTR [rsp]
14640        vpshufb	xmm0, xmm0, OWORD PTR L_avx2_aes_gcm_bswap_mask
14641        vpxor	xmm4, xmm4, xmm0
14642        ; ghash_gfmul_avx
14643        vpclmulqdq	xmm2, xmm5, xmm4, 16
14644        vpclmulqdq	xmm1, xmm5, xmm4, 1
14645        vpclmulqdq	xmm0, xmm5, xmm4, 0
14646        vpclmulqdq	xmm3, xmm5, xmm4, 17
14647        vpxor	xmm2, xmm2, xmm1
14648        vpslldq	xmm1, xmm2, 8
14649        vpsrldq	xmm2, xmm2, 8
14650        vpxor	xmm6, xmm0, xmm1
14651        vpxor	xmm4, xmm3, xmm2
14652        ; ghash_mid
14653        vpsrld	xmm0, xmm6, 31
14654        vpsrld	xmm1, xmm4, 31
14655        vpslld	xmm6, xmm6, 1
14656        vpslld	xmm4, xmm4, 1
14657        vpsrldq	xmm2, xmm0, 12
14658        vpslldq	xmm0, xmm0, 4
14659        vpslldq	xmm1, xmm1, 4
14660        vpor	xmm4, xmm4, xmm2
14661        vpor	xmm6, xmm6, xmm0
14662        vpor	xmm4, xmm4, xmm1
14663        ; ghash_red
14664        vmovdqu	xmm2, OWORD PTR L_avx2_aes_gcm_mod2_128
14665        vpclmulqdq	xmm0, xmm6, xmm2, 16
14666        vpshufd	xmm1, xmm6, 78
14667        vpxor	xmm1, xmm1, xmm0
14668        vpclmulqdq	xmm0, xmm1, xmm2, 16
14669        vpshufd	xmm1, xmm1, 78
14670        vpxor	xmm1, xmm1, xmm0
14671        vpxor	xmm4, xmm4, xmm1
14672L_AES_GCM_init_avx2_calc_iv_done:
14673        ; T = Encrypt counter
14674        vpxor	xmm0, xmm0, xmm0
14675        shl	edx, 3
14676        vmovq	xmm0, rdx
14677        vpxor	xmm4, xmm4, xmm0
14678        ; ghash_gfmul_avx
14679        vpclmulqdq	xmm2, xmm5, xmm4, 16
14680        vpclmulqdq	xmm1, xmm5, xmm4, 1
14681        vpclmulqdq	xmm0, xmm5, xmm4, 0
14682        vpclmulqdq	xmm3, xmm5, xmm4, 17
14683        vpxor	xmm2, xmm2, xmm1
14684        vpslldq	xmm1, xmm2, 8
14685        vpsrldq	xmm2, xmm2, 8
14686        vpxor	xmm6, xmm0, xmm1
14687        vpxor	xmm4, xmm3, xmm2
14688        ; ghash_mid
14689        vpsrld	xmm0, xmm6, 31
14690        vpsrld	xmm1, xmm4, 31
14691        vpslld	xmm6, xmm6, 1
14692        vpslld	xmm4, xmm4, 1
14693        vpsrldq	xmm2, xmm0, 12
14694        vpslldq	xmm0, xmm0, 4
14695        vpslldq	xmm1, xmm1, 4
14696        vpor	xmm4, xmm4, xmm2
14697        vpor	xmm6, xmm6, xmm0
14698        vpor	xmm4, xmm4, xmm1
14699        ; ghash_red
14700        vmovdqu	xmm2, OWORD PTR L_avx2_aes_gcm_mod2_128
14701        vpclmulqdq	xmm0, xmm6, xmm2, 16
14702        vpshufd	xmm1, xmm6, 78
14703        vpxor	xmm1, xmm1, xmm0
14704        vpclmulqdq	xmm0, xmm1, xmm2, 16
14705        vpshufd	xmm1, xmm1, 78
14706        vpxor	xmm1, xmm1, xmm0
14707        vpxor	xmm4, xmm4, xmm1
14708        vpshufb	xmm4, xmm4, OWORD PTR L_avx2_aes_gcm_bswap_mask
14709        ;   Encrypt counter
14710        vmovdqu	xmm7, OWORD PTR [rdi]
14711        vpxor	xmm7, xmm7, xmm4
14712        vaesenc	xmm7, xmm7, [rdi+16]
14713        vaesenc	xmm7, xmm7, [rdi+32]
14714        vaesenc	xmm7, xmm7, [rdi+48]
14715        vaesenc	xmm7, xmm7, [rdi+64]
14716        vaesenc	xmm7, xmm7, [rdi+80]
14717        vaesenc	xmm7, xmm7, [rdi+96]
14718        vaesenc	xmm7, xmm7, [rdi+112]
14719        vaesenc	xmm7, xmm7, [rdi+128]
14720        vaesenc	xmm7, xmm7, [rdi+144]
14721        cmp	esi, 11
14722        vmovdqu	xmm0, OWORD PTR [rdi+160]
14723        jl	L_AES_GCM_init_avx2_calc_iv_2_aesenc_avx_last
14724        vaesenc	xmm7, xmm7, xmm0
14725        vaesenc	xmm7, xmm7, [rdi+176]
14726        cmp	esi, 13
14727        vmovdqu	xmm0, OWORD PTR [rdi+192]
14728        jl	L_AES_GCM_init_avx2_calc_iv_2_aesenc_avx_last
14729        vaesenc	xmm7, xmm7, xmm0
14730        vaesenc	xmm7, xmm7, [rdi+208]
14731        vmovdqu	xmm0, OWORD PTR [rdi+224]
14732L_AES_GCM_init_avx2_calc_iv_2_aesenc_avx_last:
14733        vaesenclast	xmm7, xmm7, xmm0
14734        jmp	L_AES_GCM_init_avx2_iv_done
14735L_AES_GCM_init_avx2_iv_12:
14736        ; # Calculate values when IV is 12 bytes
14737        ; Set counter based on IV
14738        vmovdqu	xmm4, OWORD PTR L_avx2_aes_gcm_bswap_one
14739        vmovdqu	xmm5, OWORD PTR [rdi]
14740        vpblendd	xmm4, xmm4, [r10], 7
14741        ; H = Encrypt X(=0) and T = Encrypt counter
14742        vmovdqu	xmm6, OWORD PTR [rdi+16]
14743        vpxor	xmm7, xmm4, xmm5
14744        vaesenc	xmm5, xmm5, xmm6
14745        vaesenc	xmm7, xmm7, xmm6
14746        vmovdqu	xmm0, OWORD PTR [rdi+32]
14747        vaesenc	xmm5, xmm5, xmm0
14748        vaesenc	xmm7, xmm7, xmm0
14749        vmovdqu	xmm0, OWORD PTR [rdi+48]
14750        vaesenc	xmm5, xmm5, xmm0
14751        vaesenc	xmm7, xmm7, xmm0
14752        vmovdqu	xmm0, OWORD PTR [rdi+64]
14753        vaesenc	xmm5, xmm5, xmm0
14754        vaesenc	xmm7, xmm7, xmm0
14755        vmovdqu	xmm0, OWORD PTR [rdi+80]
14756        vaesenc	xmm5, xmm5, xmm0
14757        vaesenc	xmm7, xmm7, xmm0
14758        vmovdqu	xmm0, OWORD PTR [rdi+96]
14759        vaesenc	xmm5, xmm5, xmm0
14760        vaesenc	xmm7, xmm7, xmm0
14761        vmovdqu	xmm0, OWORD PTR [rdi+112]
14762        vaesenc	xmm5, xmm5, xmm0
14763        vaesenc	xmm7, xmm7, xmm0
14764        vmovdqu	xmm0, OWORD PTR [rdi+128]
14765        vaesenc	xmm5, xmm5, xmm0
14766        vaesenc	xmm7, xmm7, xmm0
14767        vmovdqu	xmm0, OWORD PTR [rdi+144]
14768        vaesenc	xmm5, xmm5, xmm0
14769        vaesenc	xmm7, xmm7, xmm0
14770        cmp	esi, 11
14771        vmovdqu	xmm0, OWORD PTR [rdi+160]
14772        jl	L_AES_GCM_init_avx2_calc_iv_12_last
14773        vaesenc	xmm5, xmm5, xmm0
14774        vaesenc	xmm7, xmm7, xmm0
14775        vmovdqu	xmm0, OWORD PTR [rdi+176]
14776        vaesenc	xmm5, xmm5, xmm0
14777        vaesenc	xmm7, xmm7, xmm0
14778        cmp	esi, 13
14779        vmovdqu	xmm0, OWORD PTR [rdi+192]
14780        jl	L_AES_GCM_init_avx2_calc_iv_12_last
14781        vaesenc	xmm5, xmm5, xmm0
14782        vaesenc	xmm7, xmm7, xmm0
14783        vmovdqu	xmm0, OWORD PTR [rdi+208]
14784        vaesenc	xmm5, xmm5, xmm0
14785        vaesenc	xmm7, xmm7, xmm0
14786        vmovdqu	xmm0, OWORD PTR [rdi+224]
14787L_AES_GCM_init_avx2_calc_iv_12_last:
14788        vaesenclast	xmm5, xmm5, xmm0
14789        vaesenclast	xmm7, xmm7, xmm0
14790        vpshufb	xmm5, xmm5, OWORD PTR L_avx2_aes_gcm_bswap_mask
14791L_AES_GCM_init_avx2_iv_done:
14792        vmovdqu	OWORD PTR [r9], xmm7
14793        vpshufb	xmm4, xmm4, OWORD PTR L_avx2_aes_gcm_bswap_epi64
14794        vpaddd	xmm4, xmm4, OWORD PTR L_avx2_aes_gcm_one
14795        vmovdqu	OWORD PTR [rax], xmm5
14796        vmovdqu	OWORD PTR [r8], xmm4
14797        vzeroupper
14798        vmovdqu	xmm6, OWORD PTR [rsp+16]
14799        vmovdqu	xmm7, OWORD PTR [rsp+32]
14800        add	rsp, 48
14801        pop	r12
14802        pop	rsi
14803        pop	rdi
14804        pop	rbx
14805        ret
14806AES_GCM_init_avx2 ENDP
14807_TEXT ENDS
14808_TEXT SEGMENT READONLY PARA
14809AES_GCM_aad_update_avx2 PROC
14810        mov	rax, rcx
14811        sub	rsp, 16
14812        vmovdqu	OWORD PTR [rsp], xmm6
14813        vmovdqu	xmm4, OWORD PTR [r8]
14814        vmovdqu	xmm5, OWORD PTR [r9]
14815        xor	ecx, ecx
14816L_AES_GCM_aad_update_avx2_16_loop:
14817        vmovdqu	xmm0, OWORD PTR [rax+rcx]
14818        vpshufb	xmm0, xmm0, OWORD PTR L_avx2_aes_gcm_bswap_mask
14819        vpxor	xmm4, xmm4, xmm0
14820        ; ghash_gfmul_avx
14821        vpclmulqdq	xmm2, xmm5, xmm4, 16
14822        vpclmulqdq	xmm1, xmm5, xmm4, 1
14823        vpclmulqdq	xmm0, xmm5, xmm4, 0
14824        vpclmulqdq	xmm3, xmm5, xmm4, 17
14825        vpxor	xmm2, xmm2, xmm1
14826        vpslldq	xmm1, xmm2, 8
14827        vpsrldq	xmm2, xmm2, 8
14828        vpxor	xmm6, xmm0, xmm1
14829        vpxor	xmm4, xmm3, xmm2
14830        ; ghash_mid
14831        vpsrld	xmm0, xmm6, 31
14832        vpsrld	xmm1, xmm4, 31
14833        vpslld	xmm6, xmm6, 1
14834        vpslld	xmm4, xmm4, 1
14835        vpsrldq	xmm2, xmm0, 12
14836        vpslldq	xmm0, xmm0, 4
14837        vpslldq	xmm1, xmm1, 4
14838        vpor	xmm4, xmm4, xmm2
14839        vpor	xmm6, xmm6, xmm0
14840        vpor	xmm4, xmm4, xmm1
14841        ; ghash_red
14842        vmovdqu	xmm2, OWORD PTR L_avx2_aes_gcm_mod2_128
14843        vpclmulqdq	xmm0, xmm6, xmm2, 16
14844        vpshufd	xmm1, xmm6, 78
14845        vpxor	xmm1, xmm1, xmm0
14846        vpclmulqdq	xmm0, xmm1, xmm2, 16
14847        vpshufd	xmm1, xmm1, 78
14848        vpxor	xmm1, xmm1, xmm0
14849        vpxor	xmm4, xmm4, xmm1
14850        add	ecx, 16
14851        cmp	ecx, edx
14852        jl	L_AES_GCM_aad_update_avx2_16_loop
14853        vmovdqu	OWORD PTR [r8], xmm4
14854        vzeroupper
14855        vmovdqu	xmm6, OWORD PTR [rsp]
14856        add	rsp, 16
14857        ret
14858AES_GCM_aad_update_avx2 ENDP
14859_TEXT ENDS
14860_TEXT SEGMENT READONLY PARA
14861AES_GCM_encrypt_block_avx2 PROC
14862        mov	r10, r8
14863        mov	r11, r9
14864        mov	rax, QWORD PTR [rsp+40]
14865        sub	rsp, 152
14866        vmovdqu	xmm3, OWORD PTR [rax]
14867        ; aesenc_block
14868        vmovdqu	xmm1, xmm3
14869        vpshufb	xmm0, xmm1, OWORD PTR L_avx2_aes_gcm_bswap_epi64
14870        vpaddd	xmm1, xmm1, OWORD PTR L_avx2_aes_gcm_one
14871        vpxor	xmm0, xmm0, [rcx]
14872        vmovdqu	xmm2, OWORD PTR [rcx+16]
14873        vaesenc	xmm0, xmm0, xmm2
14874        vmovdqu	xmm2, OWORD PTR [rcx+32]
14875        vaesenc	xmm0, xmm0, xmm2
14876        vmovdqu	xmm2, OWORD PTR [rcx+48]
14877        vaesenc	xmm0, xmm0, xmm2
14878        vmovdqu	xmm2, OWORD PTR [rcx+64]
14879        vaesenc	xmm0, xmm0, xmm2
14880        vmovdqu	xmm2, OWORD PTR [rcx+80]
14881        vaesenc	xmm0, xmm0, xmm2
14882        vmovdqu	xmm2, OWORD PTR [rcx+96]
14883        vaesenc	xmm0, xmm0, xmm2
14884        vmovdqu	xmm2, OWORD PTR [rcx+112]
14885        vaesenc	xmm0, xmm0, xmm2
14886        vmovdqu	xmm2, OWORD PTR [rcx+128]
14887        vaesenc	xmm0, xmm0, xmm2
14888        vmovdqu	xmm2, OWORD PTR [rcx+144]
14889        vaesenc	xmm0, xmm0, xmm2
14890        vmovdqu	xmm3, xmm1
14891        cmp	edx, 11
14892        vmovdqu	xmm1, OWORD PTR [rcx+160]
14893        jl	L_AES_GCM_encrypt_block_avx2_aesenc_block_last
14894        vaesenc	xmm0, xmm0, xmm1
14895        vmovdqu	xmm2, OWORD PTR [rcx+176]
14896        vaesenc	xmm0, xmm0, xmm2
14897        cmp	edx, 13
14898        vmovdqu	xmm1, OWORD PTR [rcx+192]
14899        jl	L_AES_GCM_encrypt_block_avx2_aesenc_block_last
14900        vaesenc	xmm0, xmm0, xmm1
14901        vmovdqu	xmm2, OWORD PTR [rcx+208]
14902        vaesenc	xmm0, xmm0, xmm2
14903        vmovdqu	xmm1, OWORD PTR [rcx+224]
14904L_AES_GCM_encrypt_block_avx2_aesenc_block_last:
14905        vaesenclast	xmm0, xmm0, xmm1
14906        vmovdqu	xmm1, OWORD PTR [r11]
14907        vpxor	xmm0, xmm0, xmm1
14908        vmovdqu	OWORD PTR [r10], xmm0
14909        vmovdqu	OWORD PTR [rax], xmm3
14910        vzeroupper
14911        add	rsp, 152
14912        ret
14913AES_GCM_encrypt_block_avx2 ENDP
14914_TEXT ENDS
14915_TEXT SEGMENT READONLY PARA
14916AES_GCM_ghash_block_avx2 PROC
14917        sub	rsp, 16
14918        vmovdqu	OWORD PTR [rsp], xmm6
14919        vmovdqu	xmm4, OWORD PTR [rdx]
14920        vmovdqu	xmm5, OWORD PTR [r8]
14921        vmovdqu	xmm0, OWORD PTR [rcx]
14922        vpshufb	xmm0, xmm0, OWORD PTR L_avx2_aes_gcm_bswap_mask
14923        vpxor	xmm4, xmm4, xmm0
14924        ; ghash_gfmul_avx
14925        vpclmulqdq	xmm2, xmm5, xmm4, 16
14926        vpclmulqdq	xmm1, xmm5, xmm4, 1
14927        vpclmulqdq	xmm0, xmm5, xmm4, 0
14928        vpclmulqdq	xmm3, xmm5, xmm4, 17
14929        vpxor	xmm2, xmm2, xmm1
14930        vpslldq	xmm1, xmm2, 8
14931        vpsrldq	xmm2, xmm2, 8
14932        vpxor	xmm6, xmm0, xmm1
14933        vpxor	xmm4, xmm3, xmm2
14934        ; ghash_mid
14935        vpsrld	xmm0, xmm6, 31
14936        vpsrld	xmm1, xmm4, 31
14937        vpslld	xmm6, xmm6, 1
14938        vpslld	xmm4, xmm4, 1
14939        vpsrldq	xmm2, xmm0, 12
14940        vpslldq	xmm0, xmm0, 4
14941        vpslldq	xmm1, xmm1, 4
14942        vpor	xmm4, xmm4, xmm2
14943        vpor	xmm6, xmm6, xmm0
14944        vpor	xmm4, xmm4, xmm1
14945        ; ghash_red
14946        vmovdqu	xmm2, OWORD PTR L_avx2_aes_gcm_mod2_128
14947        vpclmulqdq	xmm0, xmm6, xmm2, 16
14948        vpshufd	xmm1, xmm6, 78
14949        vpxor	xmm1, xmm1, xmm0
14950        vpclmulqdq	xmm0, xmm1, xmm2, 16
14951        vpshufd	xmm1, xmm1, 78
14952        vpxor	xmm1, xmm1, xmm0
14953        vpxor	xmm4, xmm4, xmm1
14954        vmovdqu	OWORD PTR [rdx], xmm4
14955        vzeroupper
14956        vmovdqu	xmm6, OWORD PTR [rsp]
14957        add	rsp, 16
14958        ret
14959AES_GCM_ghash_block_avx2 ENDP
14960_TEXT ENDS
14961_TEXT SEGMENT READONLY PARA
14962AES_GCM_encrypt_update_avx2 PROC
14963        push	r12
14964        push	r13
14965        push	r14
14966        push	r15
14967        push	rdi
14968        mov	rax, rcx
14969        mov	r10, r8
14970        mov	r8d, edx
14971        mov	r11, r9
14972        mov	r9d, DWORD PTR [rsp+80]
14973        mov	r12, QWORD PTR [rsp+88]
14974        mov	r13, QWORD PTR [rsp+96]
14975        mov	r14, QWORD PTR [rsp+104]
14976        sub	rsp, 312
14977        vmovdqu	OWORD PTR [rsp+152], xmm6
14978        vmovdqu	OWORD PTR [rsp+168], xmm7
14979        vmovdqu	OWORD PTR [rsp+184], xmm8
14980        vmovdqu	OWORD PTR [rsp+200], xmm9
14981        vmovdqu	OWORD PTR [rsp+216], xmm10
14982        vmovdqu	OWORD PTR [rsp+232], xmm11
14983        vmovdqu	OWORD PTR [rsp+248], xmm12
14984        vmovdqu	OWORD PTR [rsp+264], xmm13
14985        vmovdqu	OWORD PTR [rsp+280], xmm14
14986        vmovdqu	OWORD PTR [rsp+296], xmm15
14987        vmovdqu	xmm6, OWORD PTR [r12]
14988        vmovdqu	xmm5, OWORD PTR [r13]
14989        vmovdqu	xmm4, OWORD PTR [r14]
14990        vpsrlq	xmm1, xmm5, 63
14991        vpsllq	xmm0, xmm5, 1
14992        vpslldq	xmm1, xmm1, 8
14993        vpor	xmm0, xmm0, xmm1
14994        vpshufd	xmm5, xmm5, 255
14995        vpsrad	xmm5, xmm5, 31
14996        vpand	xmm5, xmm5, OWORD PTR L_avx2_aes_gcm_mod2_128
14997        vpxor	xmm5, xmm5, xmm0
14998        xor	edi, edi
14999        cmp	r9d, 128
15000        mov	r15d, r9d
15001        jl	L_AES_GCM_encrypt_update_avx2_done_128
15002        and	r15d, 4294967168
15003        vmovdqu	OWORD PTR [rsp+128], xmm4
15004        vmovdqu	xmm3, OWORD PTR L_avx2_aes_gcm_mod2_128
15005        ; H ^ 1 and H ^ 2
15006        vpclmulqdq	xmm9, xmm5, xmm5, 0
15007        vpclmulqdq	xmm10, xmm5, xmm5, 17
15008        vpclmulqdq	xmm8, xmm9, xmm3, 16
15009        vpshufd	xmm9, xmm9, 78
15010        vpxor	xmm9, xmm9, xmm8
15011        vpclmulqdq	xmm8, xmm9, xmm3, 16
15012        vpshufd	xmm9, xmm9, 78
15013        vpxor	xmm9, xmm9, xmm8
15014        vpxor	xmm0, xmm10, xmm9
15015        vmovdqu	OWORD PTR [rsp], xmm5
15016        vmovdqu	OWORD PTR [rsp+16], xmm0
15017        ; H ^ 3 and H ^ 4
15018        vpclmulqdq	xmm11, xmm0, xmm5, 16
15019        vpclmulqdq	xmm10, xmm0, xmm5, 1
15020        vpclmulqdq	xmm9, xmm0, xmm5, 0
15021        vpclmulqdq	xmm12, xmm0, xmm5, 17
15022        vpclmulqdq	xmm13, xmm0, xmm0, 0
15023        vpclmulqdq	xmm14, xmm0, xmm0, 17
15024        vpxor	xmm11, xmm11, xmm10
15025        vpslldq	xmm10, xmm11, 8
15026        vpsrldq	xmm11, xmm11, 8
15027        vpxor	xmm10, xmm10, xmm9
15028        vpclmulqdq	xmm8, xmm13, xmm3, 16
15029        vpclmulqdq	xmm9, xmm10, xmm3, 16
15030        vpshufd	xmm10, xmm10, 78
15031        vpshufd	xmm13, xmm13, 78
15032        vpxor	xmm10, xmm10, xmm9
15033        vpxor	xmm13, xmm13, xmm8
15034        vpclmulqdq	xmm9, xmm10, xmm3, 16
15035        vpclmulqdq	xmm8, xmm13, xmm3, 16
15036        vpshufd	xmm10, xmm10, 78
15037        vpshufd	xmm13, xmm13, 78
15038        vpxor	xmm12, xmm12, xmm11
15039        vpxor	xmm13, xmm13, xmm8
15040        vpxor	xmm10, xmm10, xmm12
15041        vpxor	xmm2, xmm13, xmm14
15042        vpxor	xmm1, xmm10, xmm9
15043        vmovdqu	OWORD PTR [rsp+32], xmm1
15044        vmovdqu	OWORD PTR [rsp+48], xmm2
15045        ; H ^ 5 and H ^ 6
15046        vpclmulqdq	xmm11, xmm1, xmm0, 16
15047        vpclmulqdq	xmm10, xmm1, xmm0, 1
15048        vpclmulqdq	xmm9, xmm1, xmm0, 0
15049        vpclmulqdq	xmm12, xmm1, xmm0, 17
15050        vpclmulqdq	xmm13, xmm1, xmm1, 0
15051        vpclmulqdq	xmm14, xmm1, xmm1, 17
15052        vpxor	xmm11, xmm11, xmm10
15053        vpslldq	xmm10, xmm11, 8
15054        vpsrldq	xmm11, xmm11, 8
15055        vpxor	xmm10, xmm10, xmm9
15056        vpclmulqdq	xmm8, xmm13, xmm3, 16
15057        vpclmulqdq	xmm9, xmm10, xmm3, 16
15058        vpshufd	xmm10, xmm10, 78
15059        vpshufd	xmm13, xmm13, 78
15060        vpxor	xmm10, xmm10, xmm9
15061        vpxor	xmm13, xmm13, xmm8
15062        vpclmulqdq	xmm9, xmm10, xmm3, 16
15063        vpclmulqdq	xmm8, xmm13, xmm3, 16
15064        vpshufd	xmm10, xmm10, 78
15065        vpshufd	xmm13, xmm13, 78
15066        vpxor	xmm12, xmm12, xmm11
15067        vpxor	xmm13, xmm13, xmm8
15068        vpxor	xmm10, xmm10, xmm12
15069        vpxor	xmm0, xmm13, xmm14
15070        vpxor	xmm7, xmm10, xmm9
15071        vmovdqu	OWORD PTR [rsp+64], xmm7
15072        vmovdqu	OWORD PTR [rsp+80], xmm0
15073        ; H ^ 7 and H ^ 8
15074        vpclmulqdq	xmm11, xmm2, xmm1, 16
15075        vpclmulqdq	xmm10, xmm2, xmm1, 1
15076        vpclmulqdq	xmm9, xmm2, xmm1, 0
15077        vpclmulqdq	xmm12, xmm2, xmm1, 17
15078        vpclmulqdq	xmm13, xmm2, xmm2, 0
15079        vpclmulqdq	xmm14, xmm2, xmm2, 17
15080        vpxor	xmm11, xmm11, xmm10
15081        vpslldq	xmm10, xmm11, 8
15082        vpsrldq	xmm11, xmm11, 8
15083        vpxor	xmm10, xmm10, xmm9
15084        vpclmulqdq	xmm8, xmm13, xmm3, 16
15085        vpclmulqdq	xmm9, xmm10, xmm3, 16
15086        vpshufd	xmm10, xmm10, 78
15087        vpshufd	xmm13, xmm13, 78
15088        vpxor	xmm10, xmm10, xmm9
15089        vpxor	xmm13, xmm13, xmm8
15090        vpclmulqdq	xmm9, xmm10, xmm3, 16
15091        vpclmulqdq	xmm8, xmm13, xmm3, 16
15092        vpshufd	xmm10, xmm10, 78
15093        vpshufd	xmm13, xmm13, 78
15094        vpxor	xmm12, xmm12, xmm11
15095        vpxor	xmm13, xmm13, xmm8
15096        vpxor	xmm10, xmm10, xmm12
15097        vpxor	xmm0, xmm13, xmm14
15098        vpxor	xmm7, xmm10, xmm9
15099        vmovdqu	OWORD PTR [rsp+96], xmm7
15100        vmovdqu	OWORD PTR [rsp+112], xmm0
15101        ; First 128 bytes of input
15102        ; aesenc_128
15103        ; aesenc_ctr
15104        vmovdqu	xmm0, OWORD PTR [rsp+128]
15105        vmovdqu	xmm1, OWORD PTR L_avx2_aes_gcm_bswap_epi64
15106        vpaddd	xmm9, xmm0, OWORD PTR L_avx2_aes_gcm_one
15107        vpshufb	xmm8, xmm0, xmm1
15108        vpaddd	xmm10, xmm0, OWORD PTR L_avx2_aes_gcm_two
15109        vpshufb	xmm9, xmm9, xmm1
15110        vpaddd	xmm11, xmm0, OWORD PTR L_avx2_aes_gcm_three
15111        vpshufb	xmm10, xmm10, xmm1
15112        vpaddd	xmm12, xmm0, OWORD PTR L_avx2_aes_gcm_four
15113        vpshufb	xmm11, xmm11, xmm1
15114        vpaddd	xmm13, xmm0, OWORD PTR L_avx2_aes_gcm_five
15115        vpshufb	xmm12, xmm12, xmm1
15116        vpaddd	xmm14, xmm0, OWORD PTR L_avx2_aes_gcm_six
15117        vpshufb	xmm13, xmm13, xmm1
15118        vpaddd	xmm15, xmm0, OWORD PTR L_avx2_aes_gcm_seven
15119        vpshufb	xmm14, xmm14, xmm1
15120        vpaddd	xmm0, xmm0, OWORD PTR L_avx2_aes_gcm_eight
15121        vpshufb	xmm15, xmm15, xmm1
15122        ; aesenc_xor
15123        vmovdqu	xmm7, OWORD PTR [rax]
15124        vmovdqu	OWORD PTR [rsp+128], xmm0
15125        vpxor	xmm8, xmm8, xmm7
15126        vpxor	xmm9, xmm9, xmm7
15127        vpxor	xmm10, xmm10, xmm7
15128        vpxor	xmm11, xmm11, xmm7
15129        vpxor	xmm12, xmm12, xmm7
15130        vpxor	xmm13, xmm13, xmm7
15131        vpxor	xmm14, xmm14, xmm7
15132        vpxor	xmm15, xmm15, xmm7
15133        vmovdqu	xmm7, OWORD PTR [rax+16]
15134        vaesenc	xmm8, xmm8, xmm7
15135        vaesenc	xmm9, xmm9, xmm7
15136        vaesenc	xmm10, xmm10, xmm7
15137        vaesenc	xmm11, xmm11, xmm7
15138        vaesenc	xmm12, xmm12, xmm7
15139        vaesenc	xmm13, xmm13, xmm7
15140        vaesenc	xmm14, xmm14, xmm7
15141        vaesenc	xmm15, xmm15, xmm7
15142        vmovdqu	xmm7, OWORD PTR [rax+32]
15143        vaesenc	xmm8, xmm8, xmm7
15144        vaesenc	xmm9, xmm9, xmm7
15145        vaesenc	xmm10, xmm10, xmm7
15146        vaesenc	xmm11, xmm11, xmm7
15147        vaesenc	xmm12, xmm12, xmm7
15148        vaesenc	xmm13, xmm13, xmm7
15149        vaesenc	xmm14, xmm14, xmm7
15150        vaesenc	xmm15, xmm15, xmm7
15151        vmovdqu	xmm7, OWORD PTR [rax+48]
15152        vaesenc	xmm8, xmm8, xmm7
15153        vaesenc	xmm9, xmm9, xmm7
15154        vaesenc	xmm10, xmm10, xmm7
15155        vaesenc	xmm11, xmm11, xmm7
15156        vaesenc	xmm12, xmm12, xmm7
15157        vaesenc	xmm13, xmm13, xmm7
15158        vaesenc	xmm14, xmm14, xmm7
15159        vaesenc	xmm15, xmm15, xmm7
15160        vmovdqu	xmm7, OWORD PTR [rax+64]
15161        vaesenc	xmm8, xmm8, xmm7
15162        vaesenc	xmm9, xmm9, xmm7
15163        vaesenc	xmm10, xmm10, xmm7
15164        vaesenc	xmm11, xmm11, xmm7
15165        vaesenc	xmm12, xmm12, xmm7
15166        vaesenc	xmm13, xmm13, xmm7
15167        vaesenc	xmm14, xmm14, xmm7
15168        vaesenc	xmm15, xmm15, xmm7
15169        vmovdqu	xmm7, OWORD PTR [rax+80]
15170        vaesenc	xmm8, xmm8, xmm7
15171        vaesenc	xmm9, xmm9, xmm7
15172        vaesenc	xmm10, xmm10, xmm7
15173        vaesenc	xmm11, xmm11, xmm7
15174        vaesenc	xmm12, xmm12, xmm7
15175        vaesenc	xmm13, xmm13, xmm7
15176        vaesenc	xmm14, xmm14, xmm7
15177        vaesenc	xmm15, xmm15, xmm7
15178        vmovdqu	xmm7, OWORD PTR [rax+96]
15179        vaesenc	xmm8, xmm8, xmm7
15180        vaesenc	xmm9, xmm9, xmm7
15181        vaesenc	xmm10, xmm10, xmm7
15182        vaesenc	xmm11, xmm11, xmm7
15183        vaesenc	xmm12, xmm12, xmm7
15184        vaesenc	xmm13, xmm13, xmm7
15185        vaesenc	xmm14, xmm14, xmm7
15186        vaesenc	xmm15, xmm15, xmm7
15187        vmovdqu	xmm7, OWORD PTR [rax+112]
15188        vaesenc	xmm8, xmm8, xmm7
15189        vaesenc	xmm9, xmm9, xmm7
15190        vaesenc	xmm10, xmm10, xmm7
15191        vaesenc	xmm11, xmm11, xmm7
15192        vaesenc	xmm12, xmm12, xmm7
15193        vaesenc	xmm13, xmm13, xmm7
15194        vaesenc	xmm14, xmm14, xmm7
15195        vaesenc	xmm15, xmm15, xmm7
15196        vmovdqu	xmm7, OWORD PTR [rax+128]
15197        vaesenc	xmm8, xmm8, xmm7
15198        vaesenc	xmm9, xmm9, xmm7
15199        vaesenc	xmm10, xmm10, xmm7
15200        vaesenc	xmm11, xmm11, xmm7
15201        vaesenc	xmm12, xmm12, xmm7
15202        vaesenc	xmm13, xmm13, xmm7
15203        vaesenc	xmm14, xmm14, xmm7
15204        vaesenc	xmm15, xmm15, xmm7
15205        vmovdqu	xmm7, OWORD PTR [rax+144]
15206        vaesenc	xmm8, xmm8, xmm7
15207        vaesenc	xmm9, xmm9, xmm7
15208        vaesenc	xmm10, xmm10, xmm7
15209        vaesenc	xmm11, xmm11, xmm7
15210        vaesenc	xmm12, xmm12, xmm7
15211        vaesenc	xmm13, xmm13, xmm7
15212        vaesenc	xmm14, xmm14, xmm7
15213        vaesenc	xmm15, xmm15, xmm7
15214        cmp	r8d, 11
15215        vmovdqu	xmm7, OWORD PTR [rax+160]
15216        jl	L_AES_GCM_encrypt_update_avx2_aesenc_128_enc_done
15217        vaesenc	xmm8, xmm8, xmm7
15218        vaesenc	xmm9, xmm9, xmm7
15219        vaesenc	xmm10, xmm10, xmm7
15220        vaesenc	xmm11, xmm11, xmm7
15221        vaesenc	xmm12, xmm12, xmm7
15222        vaesenc	xmm13, xmm13, xmm7
15223        vaesenc	xmm14, xmm14, xmm7
15224        vaesenc	xmm15, xmm15, xmm7
15225        vmovdqu	xmm7, OWORD PTR [rax+176]
15226        vaesenc	xmm8, xmm8, xmm7
15227        vaesenc	xmm9, xmm9, xmm7
15228        vaesenc	xmm10, xmm10, xmm7
15229        vaesenc	xmm11, xmm11, xmm7
15230        vaesenc	xmm12, xmm12, xmm7
15231        vaesenc	xmm13, xmm13, xmm7
15232        vaesenc	xmm14, xmm14, xmm7
15233        vaesenc	xmm15, xmm15, xmm7
15234        cmp	r8d, 13
15235        vmovdqu	xmm7, OWORD PTR [rax+192]
15236        jl	L_AES_GCM_encrypt_update_avx2_aesenc_128_enc_done
15237        vaesenc	xmm8, xmm8, xmm7
15238        vaesenc	xmm9, xmm9, xmm7
15239        vaesenc	xmm10, xmm10, xmm7
15240        vaesenc	xmm11, xmm11, xmm7
15241        vaesenc	xmm12, xmm12, xmm7
15242        vaesenc	xmm13, xmm13, xmm7
15243        vaesenc	xmm14, xmm14, xmm7
15244        vaesenc	xmm15, xmm15, xmm7
15245        vmovdqu	xmm7, OWORD PTR [rax+208]
15246        vaesenc	xmm8, xmm8, xmm7
15247        vaesenc	xmm9, xmm9, xmm7
15248        vaesenc	xmm10, xmm10, xmm7
15249        vaesenc	xmm11, xmm11, xmm7
15250        vaesenc	xmm12, xmm12, xmm7
15251        vaesenc	xmm13, xmm13, xmm7
15252        vaesenc	xmm14, xmm14, xmm7
15253        vaesenc	xmm15, xmm15, xmm7
15254        vmovdqu	xmm7, OWORD PTR [rax+224]
15255L_AES_GCM_encrypt_update_avx2_aesenc_128_enc_done:
15256        ; aesenc_last
15257        vaesenclast	xmm8, xmm8, xmm7
15258        vaesenclast	xmm9, xmm9, xmm7
15259        vaesenclast	xmm10, xmm10, xmm7
15260        vaesenclast	xmm11, xmm11, xmm7
15261        vmovdqu	xmm0, OWORD PTR [r11]
15262        vmovdqu	xmm1, OWORD PTR [r11+16]
15263        vmovdqu	xmm2, OWORD PTR [r11+32]
15264        vmovdqu	xmm3, OWORD PTR [r11+48]
15265        vpxor	xmm8, xmm8, xmm0
15266        vpxor	xmm9, xmm9, xmm1
15267        vpxor	xmm10, xmm10, xmm2
15268        vpxor	xmm11, xmm11, xmm3
15269        vmovdqu	OWORD PTR [r10], xmm8
15270        vmovdqu	OWORD PTR [r10+16], xmm9
15271        vmovdqu	OWORD PTR [r10+32], xmm10
15272        vmovdqu	OWORD PTR [r10+48], xmm11
15273        vaesenclast	xmm12, xmm12, xmm7
15274        vaesenclast	xmm13, xmm13, xmm7
15275        vaesenclast	xmm14, xmm14, xmm7
15276        vaesenclast	xmm15, xmm15, xmm7
15277        vmovdqu	xmm0, OWORD PTR [r11+64]
15278        vmovdqu	xmm1, OWORD PTR [r11+80]
15279        vmovdqu	xmm2, OWORD PTR [r11+96]
15280        vmovdqu	xmm3, OWORD PTR [r11+112]
15281        vpxor	xmm12, xmm12, xmm0
15282        vpxor	xmm13, xmm13, xmm1
15283        vpxor	xmm14, xmm14, xmm2
15284        vpxor	xmm15, xmm15, xmm3
15285        vmovdqu	OWORD PTR [r10+64], xmm12
15286        vmovdqu	OWORD PTR [r10+80], xmm13
15287        vmovdqu	OWORD PTR [r10+96], xmm14
15288        vmovdqu	OWORD PTR [r10+112], xmm15
15289        cmp	r15d, 128
15290        mov	edi, 128
15291        jle	L_AES_GCM_encrypt_update_avx2_end_128
15292        ; More 128 bytes of input
15293L_AES_GCM_encrypt_update_avx2_ghash_128:
15294        ; aesenc_128_ghash
15295        lea	rcx, QWORD PTR [r11+rdi]
15296        lea	rdx, QWORD PTR [r10+rdi]
15297        ; aesenc_ctr
15298        vmovdqu	xmm0, OWORD PTR [rsp+128]
15299        vmovdqu	xmm1, OWORD PTR L_avx2_aes_gcm_bswap_epi64
15300        vpaddd	xmm9, xmm0, OWORD PTR L_avx2_aes_gcm_one
15301        vpshufb	xmm8, xmm0, xmm1
15302        vpaddd	xmm10, xmm0, OWORD PTR L_avx2_aes_gcm_two
15303        vpshufb	xmm9, xmm9, xmm1
15304        vpaddd	xmm11, xmm0, OWORD PTR L_avx2_aes_gcm_three
15305        vpshufb	xmm10, xmm10, xmm1
15306        vpaddd	xmm12, xmm0, OWORD PTR L_avx2_aes_gcm_four
15307        vpshufb	xmm11, xmm11, xmm1
15308        vpaddd	xmm13, xmm0, OWORD PTR L_avx2_aes_gcm_five
15309        vpshufb	xmm12, xmm12, xmm1
15310        vpaddd	xmm14, xmm0, OWORD PTR L_avx2_aes_gcm_six
15311        vpshufb	xmm13, xmm13, xmm1
15312        vpaddd	xmm15, xmm0, OWORD PTR L_avx2_aes_gcm_seven
15313        vpshufb	xmm14, xmm14, xmm1
15314        vpaddd	xmm0, xmm0, OWORD PTR L_avx2_aes_gcm_eight
15315        vpshufb	xmm15, xmm15, xmm1
15316        ; aesenc_xor
15317        vmovdqu	xmm7, OWORD PTR [rax]
15318        vmovdqu	OWORD PTR [rsp+128], xmm0
15319        vpxor	xmm8, xmm8, xmm7
15320        vpxor	xmm9, xmm9, xmm7
15321        vpxor	xmm10, xmm10, xmm7
15322        vpxor	xmm11, xmm11, xmm7
15323        vpxor	xmm12, xmm12, xmm7
15324        vpxor	xmm13, xmm13, xmm7
15325        vpxor	xmm14, xmm14, xmm7
15326        vpxor	xmm15, xmm15, xmm7
15327        ; aesenc_pclmul_1
15328        vmovdqu	xmm1, OWORD PTR [rdx+-128]
15329        vmovdqu	xmm0, OWORD PTR [rax+16]
15330        vpshufb	xmm1, xmm1, OWORD PTR L_avx2_aes_gcm_bswap_mask
15331        vmovdqu	xmm2, OWORD PTR [rsp+112]
15332        vpxor	xmm1, xmm1, xmm6
15333        vpclmulqdq	xmm5, xmm1, xmm2, 16
15334        vpclmulqdq	xmm3, xmm1, xmm2, 1
15335        vpclmulqdq	xmm6, xmm1, xmm2, 0
15336        vpclmulqdq	xmm7, xmm1, xmm2, 17
15337        vaesenc	xmm8, xmm8, xmm0
15338        vaesenc	xmm9, xmm9, xmm0
15339        vaesenc	xmm10, xmm10, xmm0
15340        vaesenc	xmm11, xmm11, xmm0
15341        vaesenc	xmm12, xmm12, xmm0
15342        vaesenc	xmm13, xmm13, xmm0
15343        vaesenc	xmm14, xmm14, xmm0
15344        vaesenc	xmm15, xmm15, xmm0
15345        ; aesenc_pclmul_2
15346        vmovdqu	xmm1, OWORD PTR [rdx+-112]
15347        vmovdqu	xmm0, OWORD PTR [rsp+96]
15348        vpshufb	xmm1, xmm1, OWORD PTR L_avx2_aes_gcm_bswap_mask
15349        vpxor	xmm5, xmm5, xmm3
15350        vpclmulqdq	xmm2, xmm1, xmm0, 16
15351        vpclmulqdq	xmm3, xmm1, xmm0, 1
15352        vpclmulqdq	xmm4, xmm1, xmm0, 0
15353        vpclmulqdq	xmm1, xmm1, xmm0, 17
15354        vmovdqu	xmm0, OWORD PTR [rax+32]
15355        vpxor	xmm7, xmm7, xmm1
15356        vaesenc	xmm8, xmm8, xmm0
15357        vaesenc	xmm9, xmm9, xmm0
15358        vaesenc	xmm10, xmm10, xmm0
15359        vaesenc	xmm11, xmm11, xmm0
15360        vaesenc	xmm12, xmm12, xmm0
15361        vaesenc	xmm13, xmm13, xmm0
15362        vaesenc	xmm14, xmm14, xmm0
15363        vaesenc	xmm15, xmm15, xmm0
15364        ; aesenc_pclmul_n
15365        vmovdqu	xmm1, OWORD PTR [rdx+-96]
15366        vmovdqu	xmm0, OWORD PTR [rsp+80]
15367        vpshufb	xmm1, xmm1, OWORD PTR L_avx2_aes_gcm_bswap_mask
15368        vpxor	xmm5, xmm5, xmm2
15369        vpclmulqdq	xmm2, xmm1, xmm0, 16
15370        vpxor	xmm5, xmm5, xmm3
15371        vpclmulqdq	xmm3, xmm1, xmm0, 1
15372        vpxor	xmm6, xmm6, xmm4
15373        vpclmulqdq	xmm4, xmm1, xmm0, 0
15374        vpclmulqdq	xmm1, xmm1, xmm0, 17
15375        vmovdqu	xmm0, OWORD PTR [rax+48]
15376        vpxor	xmm7, xmm7, xmm1
15377        vaesenc	xmm8, xmm8, xmm0
15378        vaesenc	xmm9, xmm9, xmm0
15379        vaesenc	xmm10, xmm10, xmm0
15380        vaesenc	xmm11, xmm11, xmm0
15381        vaesenc	xmm12, xmm12, xmm0
15382        vaesenc	xmm13, xmm13, xmm0
15383        vaesenc	xmm14, xmm14, xmm0
15384        vaesenc	xmm15, xmm15, xmm0
15385        ; aesenc_pclmul_n
15386        vmovdqu	xmm1, OWORD PTR [rdx+-80]
15387        vmovdqu	xmm0, OWORD PTR [rsp+64]
15388        vpshufb	xmm1, xmm1, OWORD PTR L_avx2_aes_gcm_bswap_mask
15389        vpxor	xmm5, xmm5, xmm2
15390        vpclmulqdq	xmm2, xmm1, xmm0, 16
15391        vpxor	xmm5, xmm5, xmm3
15392        vpclmulqdq	xmm3, xmm1, xmm0, 1
15393        vpxor	xmm6, xmm6, xmm4
15394        vpclmulqdq	xmm4, xmm1, xmm0, 0
15395        vpclmulqdq	xmm1, xmm1, xmm0, 17
15396        vmovdqu	xmm0, OWORD PTR [rax+64]
15397        vpxor	xmm7, xmm7, xmm1
15398        vaesenc	xmm8, xmm8, xmm0
15399        vaesenc	xmm9, xmm9, xmm0
15400        vaesenc	xmm10, xmm10, xmm0
15401        vaesenc	xmm11, xmm11, xmm0
15402        vaesenc	xmm12, xmm12, xmm0
15403        vaesenc	xmm13, xmm13, xmm0
15404        vaesenc	xmm14, xmm14, xmm0
15405        vaesenc	xmm15, xmm15, xmm0
15406        ; aesenc_pclmul_n
15407        vmovdqu	xmm1, OWORD PTR [rdx+-64]
15408        vmovdqu	xmm0, OWORD PTR [rsp+48]
15409        vpshufb	xmm1, xmm1, OWORD PTR L_avx2_aes_gcm_bswap_mask
15410        vpxor	xmm5, xmm5, xmm2
15411        vpclmulqdq	xmm2, xmm1, xmm0, 16
15412        vpxor	xmm5, xmm5, xmm3
15413        vpclmulqdq	xmm3, xmm1, xmm0, 1
15414        vpxor	xmm6, xmm6, xmm4
15415        vpclmulqdq	xmm4, xmm1, xmm0, 0
15416        vpclmulqdq	xmm1, xmm1, xmm0, 17
15417        vmovdqu	xmm0, OWORD PTR [rax+80]
15418        vpxor	xmm7, xmm7, xmm1
15419        vaesenc	xmm8, xmm8, xmm0
15420        vaesenc	xmm9, xmm9, xmm0
15421        vaesenc	xmm10, xmm10, xmm0
15422        vaesenc	xmm11, xmm11, xmm0
15423        vaesenc	xmm12, xmm12, xmm0
15424        vaesenc	xmm13, xmm13, xmm0
15425        vaesenc	xmm14, xmm14, xmm0
15426        vaesenc	xmm15, xmm15, xmm0
15427        ; aesenc_pclmul_n
15428        vmovdqu	xmm1, OWORD PTR [rdx+-48]
15429        vmovdqu	xmm0, OWORD PTR [rsp+32]
15430        vpshufb	xmm1, xmm1, OWORD PTR L_avx2_aes_gcm_bswap_mask
15431        vpxor	xmm5, xmm5, xmm2
15432        vpclmulqdq	xmm2, xmm1, xmm0, 16
15433        vpxor	xmm5, xmm5, xmm3
15434        vpclmulqdq	xmm3, xmm1, xmm0, 1
15435        vpxor	xmm6, xmm6, xmm4
15436        vpclmulqdq	xmm4, xmm1, xmm0, 0
15437        vpclmulqdq	xmm1, xmm1, xmm0, 17
15438        vmovdqu	xmm0, OWORD PTR [rax+96]
15439        vpxor	xmm7, xmm7, xmm1
15440        vaesenc	xmm8, xmm8, xmm0
15441        vaesenc	xmm9, xmm9, xmm0
15442        vaesenc	xmm10, xmm10, xmm0
15443        vaesenc	xmm11, xmm11, xmm0
15444        vaesenc	xmm12, xmm12, xmm0
15445        vaesenc	xmm13, xmm13, xmm0
15446        vaesenc	xmm14, xmm14, xmm0
15447        vaesenc	xmm15, xmm15, xmm0
15448        ; aesenc_pclmul_n
15449        vmovdqu	xmm1, OWORD PTR [rdx+-32]
15450        vmovdqu	xmm0, OWORD PTR [rsp+16]
15451        vpshufb	xmm1, xmm1, OWORD PTR L_avx2_aes_gcm_bswap_mask
15452        vpxor	xmm5, xmm5, xmm2
15453        vpclmulqdq	xmm2, xmm1, xmm0, 16
15454        vpxor	xmm5, xmm5, xmm3
15455        vpclmulqdq	xmm3, xmm1, xmm0, 1
15456        vpxor	xmm6, xmm6, xmm4
15457        vpclmulqdq	xmm4, xmm1, xmm0, 0
15458        vpclmulqdq	xmm1, xmm1, xmm0, 17
15459        vmovdqu	xmm0, OWORD PTR [rax+112]
15460        vpxor	xmm7, xmm7, xmm1
15461        vaesenc	xmm8, xmm8, xmm0
15462        vaesenc	xmm9, xmm9, xmm0
15463        vaesenc	xmm10, xmm10, xmm0
15464        vaesenc	xmm11, xmm11, xmm0
15465        vaesenc	xmm12, xmm12, xmm0
15466        vaesenc	xmm13, xmm13, xmm0
15467        vaesenc	xmm14, xmm14, xmm0
15468        vaesenc	xmm15, xmm15, xmm0
15469        ; aesenc_pclmul_n
15470        vmovdqu	xmm1, OWORD PTR [rdx+-16]
15471        vmovdqu	xmm0, OWORD PTR [rsp]
15472        vpshufb	xmm1, xmm1, OWORD PTR L_avx2_aes_gcm_bswap_mask
15473        vpxor	xmm5, xmm5, xmm2
15474        vpclmulqdq	xmm2, xmm1, xmm0, 16
15475        vpxor	xmm5, xmm5, xmm3
15476        vpclmulqdq	xmm3, xmm1, xmm0, 1
15477        vpxor	xmm6, xmm6, xmm4
15478        vpclmulqdq	xmm4, xmm1, xmm0, 0
15479        vpclmulqdq	xmm1, xmm1, xmm0, 17
15480        vmovdqu	xmm0, OWORD PTR [rax+128]
15481        vpxor	xmm7, xmm7, xmm1
15482        vaesenc	xmm8, xmm8, xmm0
15483        vaesenc	xmm9, xmm9, xmm0
15484        vaesenc	xmm10, xmm10, xmm0
15485        vaesenc	xmm11, xmm11, xmm0
15486        vaesenc	xmm12, xmm12, xmm0
15487        vaesenc	xmm13, xmm13, xmm0
15488        vaesenc	xmm14, xmm14, xmm0
15489        vaesenc	xmm15, xmm15, xmm0
15490        ; aesenc_pclmul_l
15491        vpxor	xmm5, xmm5, xmm2
15492        vpxor	xmm6, xmm6, xmm4
15493        vpxor	xmm5, xmm5, xmm3
15494        vpslldq	xmm1, xmm5, 8
15495        vpsrldq	xmm5, xmm5, 8
15496        vmovdqu	xmm4, OWORD PTR [rax+144]
15497        vmovdqu	xmm0, OWORD PTR L_avx2_aes_gcm_mod2_128
15498        vaesenc	xmm8, xmm8, xmm4
15499        vpxor	xmm6, xmm6, xmm1
15500        vpxor	xmm7, xmm7, xmm5
15501        vpclmulqdq	xmm3, xmm6, xmm0, 16
15502        vaesenc	xmm9, xmm9, xmm4
15503        vaesenc	xmm10, xmm10, xmm4
15504        vaesenc	xmm11, xmm11, xmm4
15505        vpshufd	xmm6, xmm6, 78
15506        vpxor	xmm6, xmm6, xmm3
15507        vpclmulqdq	xmm3, xmm6, xmm0, 16
15508        vaesenc	xmm12, xmm12, xmm4
15509        vaesenc	xmm13, xmm13, xmm4
15510        vaesenc	xmm14, xmm14, xmm4
15511        vpshufd	xmm6, xmm6, 78
15512        vpxor	xmm6, xmm6, xmm3
15513        vpxor	xmm6, xmm6, xmm7
15514        vaesenc	xmm15, xmm15, xmm4
15515        cmp	r8d, 11
15516        vmovdqu	xmm7, OWORD PTR [rax+160]
15517        jl	L_AES_GCM_encrypt_update_avx2_aesenc_128_ghash_avx_done
15518        vaesenc	xmm8, xmm8, xmm7
15519        vaesenc	xmm9, xmm9, xmm7
15520        vaesenc	xmm10, xmm10, xmm7
15521        vaesenc	xmm11, xmm11, xmm7
15522        vaesenc	xmm12, xmm12, xmm7
15523        vaesenc	xmm13, xmm13, xmm7
15524        vaesenc	xmm14, xmm14, xmm7
15525        vaesenc	xmm15, xmm15, xmm7
15526        vmovdqu	xmm7, OWORD PTR [rax+176]
15527        vaesenc	xmm8, xmm8, xmm7
15528        vaesenc	xmm9, xmm9, xmm7
15529        vaesenc	xmm10, xmm10, xmm7
15530        vaesenc	xmm11, xmm11, xmm7
15531        vaesenc	xmm12, xmm12, xmm7
15532        vaesenc	xmm13, xmm13, xmm7
15533        vaesenc	xmm14, xmm14, xmm7
15534        vaesenc	xmm15, xmm15, xmm7
15535        cmp	r8d, 13
15536        vmovdqu	xmm7, OWORD PTR [rax+192]
15537        jl	L_AES_GCM_encrypt_update_avx2_aesenc_128_ghash_avx_done
15538        vaesenc	xmm8, xmm8, xmm7
15539        vaesenc	xmm9, xmm9, xmm7
15540        vaesenc	xmm10, xmm10, xmm7
15541        vaesenc	xmm11, xmm11, xmm7
15542        vaesenc	xmm12, xmm12, xmm7
15543        vaesenc	xmm13, xmm13, xmm7
15544        vaesenc	xmm14, xmm14, xmm7
15545        vaesenc	xmm15, xmm15, xmm7
15546        vmovdqu	xmm7, OWORD PTR [rax+208]
15547        vaesenc	xmm8, xmm8, xmm7
15548        vaesenc	xmm9, xmm9, xmm7
15549        vaesenc	xmm10, xmm10, xmm7
15550        vaesenc	xmm11, xmm11, xmm7
15551        vaesenc	xmm12, xmm12, xmm7
15552        vaesenc	xmm13, xmm13, xmm7
15553        vaesenc	xmm14, xmm14, xmm7
15554        vaesenc	xmm15, xmm15, xmm7
15555        vmovdqu	xmm7, OWORD PTR [rax+224]
15556L_AES_GCM_encrypt_update_avx2_aesenc_128_ghash_avx_done:
15557        ; aesenc_last
15558        vaesenclast	xmm8, xmm8, xmm7
15559        vaesenclast	xmm9, xmm9, xmm7
15560        vaesenclast	xmm10, xmm10, xmm7
15561        vaesenclast	xmm11, xmm11, xmm7
15562        vmovdqu	xmm0, OWORD PTR [rcx]
15563        vmovdqu	xmm1, OWORD PTR [rcx+16]
15564        vmovdqu	xmm2, OWORD PTR [rcx+32]
15565        vmovdqu	xmm3, OWORD PTR [rcx+48]
15566        vpxor	xmm8, xmm8, xmm0
15567        vpxor	xmm9, xmm9, xmm1
15568        vpxor	xmm10, xmm10, xmm2
15569        vpxor	xmm11, xmm11, xmm3
15570        vmovdqu	OWORD PTR [rdx], xmm8
15571        vmovdqu	OWORD PTR [rdx+16], xmm9
15572        vmovdqu	OWORD PTR [rdx+32], xmm10
15573        vmovdqu	OWORD PTR [rdx+48], xmm11
15574        vaesenclast	xmm12, xmm12, xmm7
15575        vaesenclast	xmm13, xmm13, xmm7
15576        vaesenclast	xmm14, xmm14, xmm7
15577        vaesenclast	xmm15, xmm15, xmm7
15578        vmovdqu	xmm0, OWORD PTR [rcx+64]
15579        vmovdqu	xmm1, OWORD PTR [rcx+80]
15580        vmovdqu	xmm2, OWORD PTR [rcx+96]
15581        vmovdqu	xmm3, OWORD PTR [rcx+112]
15582        vpxor	xmm12, xmm12, xmm0
15583        vpxor	xmm13, xmm13, xmm1
15584        vpxor	xmm14, xmm14, xmm2
15585        vpxor	xmm15, xmm15, xmm3
15586        vmovdqu	OWORD PTR [rdx+64], xmm12
15587        vmovdqu	OWORD PTR [rdx+80], xmm13
15588        vmovdqu	OWORD PTR [rdx+96], xmm14
15589        vmovdqu	OWORD PTR [rdx+112], xmm15
15590        ; aesenc_128_ghash - end
15591        add	edi, 128
15592        cmp	edi, r15d
15593        jl	L_AES_GCM_encrypt_update_avx2_ghash_128
15594L_AES_GCM_encrypt_update_avx2_end_128:
15595        vmovdqu	xmm4, OWORD PTR L_avx2_aes_gcm_bswap_mask
15596        vpshufb	xmm8, xmm8, xmm4
15597        vpshufb	xmm9, xmm9, xmm4
15598        vpshufb	xmm10, xmm10, xmm4
15599        vpshufb	xmm11, xmm11, xmm4
15600        vpshufb	xmm12, xmm12, xmm4
15601        vpshufb	xmm13, xmm13, xmm4
15602        vpshufb	xmm14, xmm14, xmm4
15603        vpshufb	xmm15, xmm15, xmm4
15604        vpxor	xmm8, xmm8, xmm6
15605        vmovdqu	xmm7, OWORD PTR [rsp]
15606        vpclmulqdq	xmm5, xmm7, xmm15, 16
15607        vpclmulqdq	xmm1, xmm7, xmm15, 1
15608        vpclmulqdq	xmm4, xmm7, xmm15, 0
15609        vpclmulqdq	xmm6, xmm7, xmm15, 17
15610        vpxor	xmm5, xmm5, xmm1
15611        vmovdqu	xmm7, OWORD PTR [rsp+16]
15612        vpclmulqdq	xmm2, xmm7, xmm14, 16
15613        vpclmulqdq	xmm1, xmm7, xmm14, 1
15614        vpclmulqdq	xmm0, xmm7, xmm14, 0
15615        vpclmulqdq	xmm3, xmm7, xmm14, 17
15616        vpxor	xmm2, xmm2, xmm1
15617        vpxor	xmm6, xmm6, xmm3
15618        vpxor	xmm5, xmm5, xmm2
15619        vpxor	xmm4, xmm4, xmm0
15620        vmovdqu	xmm15, OWORD PTR [rsp+32]
15621        vmovdqu	xmm7, OWORD PTR [rsp+48]
15622        vpclmulqdq	xmm2, xmm15, xmm13, 16
15623        vpclmulqdq	xmm1, xmm15, xmm13, 1
15624        vpclmulqdq	xmm0, xmm15, xmm13, 0
15625        vpclmulqdq	xmm3, xmm15, xmm13, 17
15626        vpxor	xmm2, xmm2, xmm1
15627        vpxor	xmm6, xmm6, xmm3
15628        vpxor	xmm5, xmm5, xmm2
15629        vpxor	xmm4, xmm4, xmm0
15630        vpclmulqdq	xmm2, xmm7, xmm12, 16
15631        vpclmulqdq	xmm1, xmm7, xmm12, 1
15632        vpclmulqdq	xmm0, xmm7, xmm12, 0
15633        vpclmulqdq	xmm3, xmm7, xmm12, 17
15634        vpxor	xmm2, xmm2, xmm1
15635        vpxor	xmm6, xmm6, xmm3
15636        vpxor	xmm5, xmm5, xmm2
15637        vpxor	xmm4, xmm4, xmm0
15638        vmovdqu	xmm15, OWORD PTR [rsp+64]
15639        vmovdqu	xmm7, OWORD PTR [rsp+80]
15640        vpclmulqdq	xmm2, xmm15, xmm11, 16
15641        vpclmulqdq	xmm1, xmm15, xmm11, 1
15642        vpclmulqdq	xmm0, xmm15, xmm11, 0
15643        vpclmulqdq	xmm3, xmm15, xmm11, 17
15644        vpxor	xmm2, xmm2, xmm1
15645        vpxor	xmm6, xmm6, xmm3
15646        vpxor	xmm5, xmm5, xmm2
15647        vpxor	xmm4, xmm4, xmm0
15648        vpclmulqdq	xmm2, xmm7, xmm10, 16
15649        vpclmulqdq	xmm1, xmm7, xmm10, 1
15650        vpclmulqdq	xmm0, xmm7, xmm10, 0
15651        vpclmulqdq	xmm3, xmm7, xmm10, 17
15652        vpxor	xmm2, xmm2, xmm1
15653        vpxor	xmm6, xmm6, xmm3
15654        vpxor	xmm5, xmm5, xmm2
15655        vpxor	xmm4, xmm4, xmm0
15656        vmovdqu	xmm15, OWORD PTR [rsp+96]
15657        vmovdqu	xmm7, OWORD PTR [rsp+112]
15658        vpclmulqdq	xmm2, xmm15, xmm9, 16
15659        vpclmulqdq	xmm1, xmm15, xmm9, 1
15660        vpclmulqdq	xmm0, xmm15, xmm9, 0
15661        vpclmulqdq	xmm3, xmm15, xmm9, 17
15662        vpxor	xmm2, xmm2, xmm1
15663        vpxor	xmm6, xmm6, xmm3
15664        vpxor	xmm5, xmm5, xmm2
15665        vpxor	xmm4, xmm4, xmm0
15666        vpclmulqdq	xmm2, xmm7, xmm8, 16
15667        vpclmulqdq	xmm1, xmm7, xmm8, 1
15668        vpclmulqdq	xmm0, xmm7, xmm8, 0
15669        vpclmulqdq	xmm3, xmm7, xmm8, 17
15670        vpxor	xmm2, xmm2, xmm1
15671        vpxor	xmm6, xmm6, xmm3
15672        vpxor	xmm5, xmm5, xmm2
15673        vpxor	xmm4, xmm4, xmm0
15674        vpslldq	xmm7, xmm5, 8
15675        vpsrldq	xmm5, xmm5, 8
15676        vpxor	xmm4, xmm4, xmm7
15677        vpxor	xmm6, xmm6, xmm5
15678        ; ghash_red
15679        vmovdqu	xmm2, OWORD PTR L_avx2_aes_gcm_mod2_128
15680        vpclmulqdq	xmm0, xmm4, xmm2, 16
15681        vpshufd	xmm1, xmm4, 78
15682        vpxor	xmm1, xmm1, xmm0
15683        vpclmulqdq	xmm0, xmm1, xmm2, 16
15684        vpshufd	xmm1, xmm1, 78
15685        vpxor	xmm1, xmm1, xmm0
15686        vpxor	xmm6, xmm6, xmm1
15687        vmovdqu	xmm5, OWORD PTR [rsp]
15688        vmovdqu	xmm4, OWORD PTR [rsp+128]
15689L_AES_GCM_encrypt_update_avx2_done_128:
15690        cmp	edi, r9d
15691        je	L_AES_GCM_encrypt_update_avx2_done_enc
15692        mov	r15d, r9d
15693        and	r15d, 4294967280
15694        cmp	edi, r15d
15695        jge	L_AES_GCM_encrypt_update_avx2_last_block_done
15696        ; aesenc_block
15697        vmovdqu	xmm1, xmm4
15698        vpshufb	xmm0, xmm1, OWORD PTR L_avx2_aes_gcm_bswap_epi64
15699        vpaddd	xmm1, xmm1, OWORD PTR L_avx2_aes_gcm_one
15700        vpxor	xmm0, xmm0, [rax]
15701        vmovdqu	xmm2, OWORD PTR [rax+16]
15702        vaesenc	xmm0, xmm0, xmm2
15703        vmovdqu	xmm2, OWORD PTR [rax+32]
15704        vaesenc	xmm0, xmm0, xmm2
15705        vmovdqu	xmm2, OWORD PTR [rax+48]
15706        vaesenc	xmm0, xmm0, xmm2
15707        vmovdqu	xmm2, OWORD PTR [rax+64]
15708        vaesenc	xmm0, xmm0, xmm2
15709        vmovdqu	xmm2, OWORD PTR [rax+80]
15710        vaesenc	xmm0, xmm0, xmm2
15711        vmovdqu	xmm2, OWORD PTR [rax+96]
15712        vaesenc	xmm0, xmm0, xmm2
15713        vmovdqu	xmm2, OWORD PTR [rax+112]
15714        vaesenc	xmm0, xmm0, xmm2
15715        vmovdqu	xmm2, OWORD PTR [rax+128]
15716        vaesenc	xmm0, xmm0, xmm2
15717        vmovdqu	xmm2, OWORD PTR [rax+144]
15718        vaesenc	xmm0, xmm0, xmm2
15719        vmovdqu	xmm4, xmm1
15720        cmp	r8d, 11
15721        vmovdqu	xmm1, OWORD PTR [rax+160]
15722        jl	L_AES_GCM_encrypt_update_avx2_aesenc_block_last
15723        vaesenc	xmm0, xmm0, xmm1
15724        vmovdqu	xmm2, OWORD PTR [rax+176]
15725        vaesenc	xmm0, xmm0, xmm2
15726        cmp	r8d, 13
15727        vmovdqu	xmm1, OWORD PTR [rax+192]
15728        jl	L_AES_GCM_encrypt_update_avx2_aesenc_block_last
15729        vaesenc	xmm0, xmm0, xmm1
15730        vmovdqu	xmm2, OWORD PTR [rax+208]
15731        vaesenc	xmm0, xmm0, xmm2
15732        vmovdqu	xmm1, OWORD PTR [rax+224]
15733L_AES_GCM_encrypt_update_avx2_aesenc_block_last:
15734        vaesenclast	xmm0, xmm0, xmm1
15735        vmovdqu	xmm1, OWORD PTR [r11+rdi]
15736        vpxor	xmm0, xmm0, xmm1
15737        vmovdqu	OWORD PTR [r10+rdi], xmm0
15738        vpshufb	xmm0, xmm0, OWORD PTR L_avx2_aes_gcm_bswap_mask
15739        vpxor	xmm6, xmm6, xmm0
15740        add	edi, 16
15741        cmp	edi, r15d
15742        jge	L_AES_GCM_encrypt_update_avx2_last_block_ghash
15743L_AES_GCM_encrypt_update_avx2_last_block_start:
15744        vmovdqu	xmm12, OWORD PTR [r11+rdi]
15745        vpshufb	xmm11, xmm4, OWORD PTR L_avx2_aes_gcm_bswap_epi64
15746        vpaddd	xmm4, xmm4, OWORD PTR L_avx2_aes_gcm_one
15747        ; aesenc_gfmul_sb
15748        vpclmulqdq	xmm2, xmm6, xmm5, 1
15749        vpclmulqdq	xmm3, xmm6, xmm5, 16
15750        vpclmulqdq	xmm1, xmm6, xmm5, 0
15751        vpclmulqdq	xmm8, xmm6, xmm5, 17
15752        vpxor	xmm11, xmm11, [rax]
15753        vaesenc	xmm11, xmm11, [rax+16]
15754        vpxor	xmm3, xmm3, xmm2
15755        vpslldq	xmm2, xmm3, 8
15756        vpsrldq	xmm3, xmm3, 8
15757        vaesenc	xmm11, xmm11, [rax+32]
15758        vpxor	xmm2, xmm2, xmm1
15759        vpclmulqdq	xmm1, xmm2, OWORD PTR L_avx2_aes_gcm_mod2_128, 16
15760        vaesenc	xmm11, xmm11, [rax+48]
15761        vaesenc	xmm11, xmm11, [rax+64]
15762        vaesenc	xmm11, xmm11, [rax+80]
15763        vpshufd	xmm2, xmm2, 78
15764        vpxor	xmm2, xmm2, xmm1
15765        vpclmulqdq	xmm1, xmm2, OWORD PTR L_avx2_aes_gcm_mod2_128, 16
15766        vaesenc	xmm11, xmm11, [rax+96]
15767        vaesenc	xmm11, xmm11, [rax+112]
15768        vaesenc	xmm11, xmm11, [rax+128]
15769        vpshufd	xmm2, xmm2, 78
15770        vaesenc	xmm11, xmm11, [rax+144]
15771        vpxor	xmm8, xmm8, xmm3
15772        vpxor	xmm2, xmm2, xmm8
15773        vmovdqu	xmm0, OWORD PTR [rax+160]
15774        cmp	r8d, 11
15775        jl	L_AES_GCM_encrypt_update_avx2_aesenc_gfmul_sb_last
15776        vaesenc	xmm11, xmm11, xmm0
15777        vaesenc	xmm11, xmm11, [rax+176]
15778        vmovdqu	xmm0, OWORD PTR [rax+192]
15779        cmp	r8d, 13
15780        jl	L_AES_GCM_encrypt_update_avx2_aesenc_gfmul_sb_last
15781        vaesenc	xmm11, xmm11, xmm0
15782        vaesenc	xmm11, xmm11, [rax+208]
15783        vmovdqu	xmm0, OWORD PTR [rax+224]
15784L_AES_GCM_encrypt_update_avx2_aesenc_gfmul_sb_last:
15785        vaesenclast	xmm11, xmm11, xmm0
15786        vpxor	xmm6, xmm2, xmm1
15787        vpxor	xmm11, xmm11, xmm12
15788        vmovdqu	OWORD PTR [r10+rdi], xmm11
15789        vpshufb	xmm11, xmm11, OWORD PTR L_avx2_aes_gcm_bswap_mask
15790        vpxor	xmm6, xmm6, xmm11
15791        add	edi, 16
15792        cmp	edi, r15d
15793        jl	L_AES_GCM_encrypt_update_avx2_last_block_start
15794L_AES_GCM_encrypt_update_avx2_last_block_ghash:
15795        ; ghash_gfmul_red
15796        vpclmulqdq	xmm10, xmm6, xmm5, 16
15797        vpclmulqdq	xmm9, xmm6, xmm5, 1
15798        vpclmulqdq	xmm8, xmm6, xmm5, 0
15799        vpxor	xmm10, xmm10, xmm9
15800        vpslldq	xmm9, xmm10, 8
15801        vpsrldq	xmm10, xmm10, 8
15802        vpxor	xmm9, xmm9, xmm8
15803        vpclmulqdq	xmm6, xmm6, xmm5, 17
15804        vpclmulqdq	xmm8, xmm9, OWORD PTR L_avx2_aes_gcm_mod2_128, 16
15805        vpshufd	xmm9, xmm9, 78
15806        vpxor	xmm9, xmm9, xmm8
15807        vpclmulqdq	xmm8, xmm9, OWORD PTR L_avx2_aes_gcm_mod2_128, 16
15808        vpshufd	xmm9, xmm9, 78
15809        vpxor	xmm6, xmm6, xmm10
15810        vpxor	xmm6, xmm6, xmm9
15811        vpxor	xmm6, xmm6, xmm8
15812L_AES_GCM_encrypt_update_avx2_last_block_done:
15813L_AES_GCM_encrypt_update_avx2_done_enc:
15814        vmovdqu	OWORD PTR [r12], xmm6
15815        vmovdqu	OWORD PTR [r14], xmm4
15816        vzeroupper
15817        vmovdqu	xmm6, OWORD PTR [rsp+152]
15818        vmovdqu	xmm7, OWORD PTR [rsp+168]
15819        vmovdqu	xmm8, OWORD PTR [rsp+184]
15820        vmovdqu	xmm9, OWORD PTR [rsp+200]
15821        vmovdqu	xmm10, OWORD PTR [rsp+216]
15822        vmovdqu	xmm11, OWORD PTR [rsp+232]
15823        vmovdqu	xmm12, OWORD PTR [rsp+248]
15824        vmovdqu	xmm13, OWORD PTR [rsp+264]
15825        vmovdqu	xmm14, OWORD PTR [rsp+280]
15826        vmovdqu	xmm15, OWORD PTR [rsp+296]
15827        add	rsp, 312
15828        pop	rdi
15829        pop	r15
15830        pop	r14
15831        pop	r13
15832        pop	r12
15833        ret
15834AES_GCM_encrypt_update_avx2 ENDP
15835_TEXT ENDS
15836_TEXT SEGMENT READONLY PARA
15837AES_GCM_encrypt_final_avx2 PROC
15838        push	r12
15839        push	r13
15840        mov	eax, DWORD PTR [rsp+56]
15841        mov	r10, QWORD PTR [rsp+64]
15842        mov	r11, QWORD PTR [rsp+72]
15843        sub	rsp, 48
15844        vmovdqu	OWORD PTR [rsp+16], xmm6
15845        vmovdqu	OWORD PTR [rsp+32], xmm7
15846        vmovdqu	xmm4, OWORD PTR [rcx]
15847        vmovdqu	xmm5, OWORD PTR [r10]
15848        vmovdqu	xmm6, OWORD PTR [r11]
15849        vpsrlq	xmm1, xmm5, 63
15850        vpsllq	xmm0, xmm5, 1
15851        vpslldq	xmm1, xmm1, 8
15852        vpor	xmm0, xmm0, xmm1
15853        vpshufd	xmm5, xmm5, 255
15854        vpsrad	xmm5, xmm5, 31
15855        vpand	xmm5, xmm5, OWORD PTR L_avx2_aes_gcm_mod2_128
15856        vpxor	xmm5, xmm5, xmm0
15857        ; calc_tag
15858        shl	r9, 3
15859        shl	rax, 3
15860        vmovq	xmm0, r9
15861        vmovq	xmm1, rax
15862        vpunpcklqdq	xmm0, xmm0, xmm1
15863        vpxor	xmm0, xmm0, xmm4
15864        ; ghash_gfmul_red
15865        vpclmulqdq	xmm7, xmm0, xmm5, 16
15866        vpclmulqdq	xmm3, xmm0, xmm5, 1
15867        vpclmulqdq	xmm2, xmm0, xmm5, 0
15868        vpxor	xmm7, xmm7, xmm3
15869        vpslldq	xmm3, xmm7, 8
15870        vpsrldq	xmm7, xmm7, 8
15871        vpxor	xmm3, xmm3, xmm2
15872        vpclmulqdq	xmm0, xmm0, xmm5, 17
15873        vpclmulqdq	xmm2, xmm3, OWORD PTR L_avx2_aes_gcm_mod2_128, 16
15874        vpshufd	xmm3, xmm3, 78
15875        vpxor	xmm3, xmm3, xmm2
15876        vpclmulqdq	xmm2, xmm3, OWORD PTR L_avx2_aes_gcm_mod2_128, 16
15877        vpshufd	xmm3, xmm3, 78
15878        vpxor	xmm0, xmm0, xmm7
15879        vpxor	xmm0, xmm0, xmm3
15880        vpxor	xmm0, xmm0, xmm2
15881        vpshufb	xmm0, xmm0, OWORD PTR L_avx2_aes_gcm_bswap_mask
15882        vpxor	xmm0, xmm0, xmm6
15883        ; store_tag
15884        cmp	r8d, 16
15885        je	L_AES_GCM_encrypt_final_avx2_store_tag_16
15886        xor	r12, r12
15887        vmovdqu	OWORD PTR [rsp], xmm0
15888L_AES_GCM_encrypt_final_avx2_store_tag_loop:
15889        movzx	r13d, BYTE PTR [rsp+r12]
15890        mov	BYTE PTR [rdx+r12], r13b
15891        inc	r12d
15892        cmp	r12d, r8d
15893        jne	L_AES_GCM_encrypt_final_avx2_store_tag_loop
15894        jmp	L_AES_GCM_encrypt_final_avx2_store_tag_done
15895L_AES_GCM_encrypt_final_avx2_store_tag_16:
15896        vmovdqu	OWORD PTR [rdx], xmm0
15897L_AES_GCM_encrypt_final_avx2_store_tag_done:
15898        vzeroupper
15899        vmovdqu	xmm6, OWORD PTR [rsp+16]
15900        vmovdqu	xmm7, OWORD PTR [rsp+32]
15901        add	rsp, 48
15902        pop	r13
15903        pop	r12
15904        ret
15905AES_GCM_encrypt_final_avx2 ENDP
15906_TEXT ENDS
15907_TEXT SEGMENT READONLY PARA
15908AES_GCM_decrypt_update_avx2 PROC
15909        push	r13
15910        push	r12
15911        push	r14
15912        push	r15
15913        push	rdi
15914        mov	rax, rcx
15915        mov	r10, r8
15916        mov	r8d, edx
15917        mov	r11, r9
15918        mov	r9d, DWORD PTR [rsp+80]
15919        mov	r12, QWORD PTR [rsp+88]
15920        mov	r14, QWORD PTR [rsp+96]
15921        mov	r15, QWORD PTR [rsp+104]
15922        sub	rsp, 328
15923        vmovdqu	OWORD PTR [rsp+168], xmm6
15924        vmovdqu	OWORD PTR [rsp+184], xmm7
15925        vmovdqu	OWORD PTR [rsp+200], xmm8
15926        vmovdqu	OWORD PTR [rsp+216], xmm9
15927        vmovdqu	OWORD PTR [rsp+232], xmm10
15928        vmovdqu	OWORD PTR [rsp+248], xmm11
15929        vmovdqu	OWORD PTR [rsp+264], xmm12
15930        vmovdqu	OWORD PTR [rsp+280], xmm13
15931        vmovdqu	OWORD PTR [rsp+296], xmm14
15932        vmovdqu	OWORD PTR [rsp+312], xmm15
15933        vmovdqu	xmm6, OWORD PTR [r12]
15934        vmovdqu	xmm5, OWORD PTR [r14]
15935        vmovdqu	xmm4, OWORD PTR [r15]
15936        ; Calculate H
15937        vpsrlq	xmm1, xmm5, 63
15938        vpsllq	xmm0, xmm5, 1
15939        vpslldq	xmm1, xmm1, 8
15940        vpor	xmm0, xmm0, xmm1
15941        vpshufd	xmm5, xmm5, 255
15942        vpsrad	xmm5, xmm5, 31
15943        vpand	xmm5, xmm5, OWORD PTR L_avx2_aes_gcm_mod2_128
15944        vpxor	xmm5, xmm5, xmm0
15945        xor	edi, edi
15946        cmp	r9d, 128
15947        mov	r13d, r9d
15948        jl	L_AES_GCM_decrypt_update_avx2_done_128
15949        and	r13d, 4294967168
15950        vmovdqu	OWORD PTR [rsp+128], xmm4
15951        vmovdqu	OWORD PTR [rsp+144], xmm15
15952        vmovdqu	xmm3, OWORD PTR L_avx2_aes_gcm_mod2_128
15953        ; H ^ 1 and H ^ 2
15954        vpclmulqdq	xmm9, xmm5, xmm5, 0
15955        vpclmulqdq	xmm10, xmm5, xmm5, 17
15956        vpclmulqdq	xmm8, xmm9, xmm3, 16
15957        vpshufd	xmm9, xmm9, 78
15958        vpxor	xmm9, xmm9, xmm8
15959        vpclmulqdq	xmm8, xmm9, xmm3, 16
15960        vpshufd	xmm9, xmm9, 78
15961        vpxor	xmm9, xmm9, xmm8
15962        vpxor	xmm0, xmm10, xmm9
15963        vmovdqu	OWORD PTR [rsp], xmm5
15964        vmovdqu	OWORD PTR [rsp+16], xmm0
15965        ; H ^ 3 and H ^ 4
15966        vpclmulqdq	xmm11, xmm0, xmm5, 16
15967        vpclmulqdq	xmm10, xmm0, xmm5, 1
15968        vpclmulqdq	xmm9, xmm0, xmm5, 0
15969        vpclmulqdq	xmm12, xmm0, xmm5, 17
15970        vpclmulqdq	xmm13, xmm0, xmm0, 0
15971        vpclmulqdq	xmm14, xmm0, xmm0, 17
15972        vpxor	xmm11, xmm11, xmm10
15973        vpslldq	xmm10, xmm11, 8
15974        vpsrldq	xmm11, xmm11, 8
15975        vpxor	xmm10, xmm10, xmm9
15976        vpclmulqdq	xmm8, xmm13, xmm3, 16
15977        vpclmulqdq	xmm9, xmm10, xmm3, 16
15978        vpshufd	xmm10, xmm10, 78
15979        vpshufd	xmm13, xmm13, 78
15980        vpxor	xmm10, xmm10, xmm9
15981        vpxor	xmm13, xmm13, xmm8
15982        vpclmulqdq	xmm9, xmm10, xmm3, 16
15983        vpclmulqdq	xmm8, xmm13, xmm3, 16
15984        vpshufd	xmm10, xmm10, 78
15985        vpshufd	xmm13, xmm13, 78
15986        vpxor	xmm12, xmm12, xmm11
15987        vpxor	xmm13, xmm13, xmm8
15988        vpxor	xmm10, xmm10, xmm12
15989        vpxor	xmm2, xmm13, xmm14
15990        vpxor	xmm1, xmm10, xmm9
15991        vmovdqu	OWORD PTR [rsp+32], xmm1
15992        vmovdqu	OWORD PTR [rsp+48], xmm2
15993        ; H ^ 5 and H ^ 6
15994        vpclmulqdq	xmm11, xmm1, xmm0, 16
15995        vpclmulqdq	xmm10, xmm1, xmm0, 1
15996        vpclmulqdq	xmm9, xmm1, xmm0, 0
15997        vpclmulqdq	xmm12, xmm1, xmm0, 17
15998        vpclmulqdq	xmm13, xmm1, xmm1, 0
15999        vpclmulqdq	xmm14, xmm1, xmm1, 17
16000        vpxor	xmm11, xmm11, xmm10
16001        vpslldq	xmm10, xmm11, 8
16002        vpsrldq	xmm11, xmm11, 8
16003        vpxor	xmm10, xmm10, xmm9
16004        vpclmulqdq	xmm8, xmm13, xmm3, 16
16005        vpclmulqdq	xmm9, xmm10, xmm3, 16
16006        vpshufd	xmm10, xmm10, 78
16007        vpshufd	xmm13, xmm13, 78
16008        vpxor	xmm10, xmm10, xmm9
16009        vpxor	xmm13, xmm13, xmm8
16010        vpclmulqdq	xmm9, xmm10, xmm3, 16
16011        vpclmulqdq	xmm8, xmm13, xmm3, 16
16012        vpshufd	xmm10, xmm10, 78
16013        vpshufd	xmm13, xmm13, 78
16014        vpxor	xmm12, xmm12, xmm11
16015        vpxor	xmm13, xmm13, xmm8
16016        vpxor	xmm10, xmm10, xmm12
16017        vpxor	xmm0, xmm13, xmm14
16018        vpxor	xmm7, xmm10, xmm9
16019        vmovdqu	OWORD PTR [rsp+64], xmm7
16020        vmovdqu	OWORD PTR [rsp+80], xmm0
16021        ; H ^ 7 and H ^ 8
16022        vpclmulqdq	xmm11, xmm2, xmm1, 16
16023        vpclmulqdq	xmm10, xmm2, xmm1, 1
16024        vpclmulqdq	xmm9, xmm2, xmm1, 0
16025        vpclmulqdq	xmm12, xmm2, xmm1, 17
16026        vpclmulqdq	xmm13, xmm2, xmm2, 0
16027        vpclmulqdq	xmm14, xmm2, xmm2, 17
16028        vpxor	xmm11, xmm11, xmm10
16029        vpslldq	xmm10, xmm11, 8
16030        vpsrldq	xmm11, xmm11, 8
16031        vpxor	xmm10, xmm10, xmm9
16032        vpclmulqdq	xmm8, xmm13, xmm3, 16
16033        vpclmulqdq	xmm9, xmm10, xmm3, 16
16034        vpshufd	xmm10, xmm10, 78
16035        vpshufd	xmm13, xmm13, 78
16036        vpxor	xmm10, xmm10, xmm9
16037        vpxor	xmm13, xmm13, xmm8
16038        vpclmulqdq	xmm9, xmm10, xmm3, 16
16039        vpclmulqdq	xmm8, xmm13, xmm3, 16
16040        vpshufd	xmm10, xmm10, 78
16041        vpshufd	xmm13, xmm13, 78
16042        vpxor	xmm12, xmm12, xmm11
16043        vpxor	xmm13, xmm13, xmm8
16044        vpxor	xmm10, xmm10, xmm12
16045        vpxor	xmm0, xmm13, xmm14
16046        vpxor	xmm7, xmm10, xmm9
16047        vmovdqu	OWORD PTR [rsp+96], xmm7
16048        vmovdqu	OWORD PTR [rsp+112], xmm0
16049L_AES_GCM_decrypt_update_avx2_ghash_128:
16050        ; aesenc_128_ghash
16051        lea	rcx, QWORD PTR [r11+rdi]
16052        lea	rdx, QWORD PTR [r10+rdi]
16053        ; aesenc_ctr
16054        vmovdqu	xmm0, OWORD PTR [rsp+128]
16055        vmovdqu	xmm1, OWORD PTR L_avx2_aes_gcm_bswap_epi64
16056        vpaddd	xmm9, xmm0, OWORD PTR L_avx2_aes_gcm_one
16057        vpshufb	xmm8, xmm0, xmm1
16058        vpaddd	xmm10, xmm0, OWORD PTR L_avx2_aes_gcm_two
16059        vpshufb	xmm9, xmm9, xmm1
16060        vpaddd	xmm11, xmm0, OWORD PTR L_avx2_aes_gcm_three
16061        vpshufb	xmm10, xmm10, xmm1
16062        vpaddd	xmm12, xmm0, OWORD PTR L_avx2_aes_gcm_four
16063        vpshufb	xmm11, xmm11, xmm1
16064        vpaddd	xmm13, xmm0, OWORD PTR L_avx2_aes_gcm_five
16065        vpshufb	xmm12, xmm12, xmm1
16066        vpaddd	xmm14, xmm0, OWORD PTR L_avx2_aes_gcm_six
16067        vpshufb	xmm13, xmm13, xmm1
16068        vpaddd	xmm15, xmm0, OWORD PTR L_avx2_aes_gcm_seven
16069        vpshufb	xmm14, xmm14, xmm1
16070        vpaddd	xmm0, xmm0, OWORD PTR L_avx2_aes_gcm_eight
16071        vpshufb	xmm15, xmm15, xmm1
16072        ; aesenc_xor
16073        vmovdqu	xmm7, OWORD PTR [rax]
16074        vmovdqu	OWORD PTR [rsp+128], xmm0
16075        vpxor	xmm8, xmm8, xmm7
16076        vpxor	xmm9, xmm9, xmm7
16077        vpxor	xmm10, xmm10, xmm7
16078        vpxor	xmm11, xmm11, xmm7
16079        vpxor	xmm12, xmm12, xmm7
16080        vpxor	xmm13, xmm13, xmm7
16081        vpxor	xmm14, xmm14, xmm7
16082        vpxor	xmm15, xmm15, xmm7
16083        ; aesenc_pclmul_1
16084        vmovdqu	xmm1, OWORD PTR [rcx]
16085        vmovdqu	xmm0, OWORD PTR [rax+16]
16086        vpshufb	xmm1, xmm1, OWORD PTR L_avx2_aes_gcm_bswap_mask
16087        vmovdqu	xmm2, OWORD PTR [rsp+112]
16088        vpxor	xmm1, xmm1, xmm6
16089        vpclmulqdq	xmm5, xmm1, xmm2, 16
16090        vpclmulqdq	xmm3, xmm1, xmm2, 1
16091        vpclmulqdq	xmm6, xmm1, xmm2, 0
16092        vpclmulqdq	xmm7, xmm1, xmm2, 17
16093        vaesenc	xmm8, xmm8, xmm0
16094        vaesenc	xmm9, xmm9, xmm0
16095        vaesenc	xmm10, xmm10, xmm0
16096        vaesenc	xmm11, xmm11, xmm0
16097        vaesenc	xmm12, xmm12, xmm0
16098        vaesenc	xmm13, xmm13, xmm0
16099        vaesenc	xmm14, xmm14, xmm0
16100        vaesenc	xmm15, xmm15, xmm0
16101        ; aesenc_pclmul_2
16102        vmovdqu	xmm1, OWORD PTR [rcx+16]
16103        vmovdqu	xmm0, OWORD PTR [rsp+96]
16104        vpshufb	xmm1, xmm1, OWORD PTR L_avx2_aes_gcm_bswap_mask
16105        vpxor	xmm5, xmm5, xmm3
16106        vpclmulqdq	xmm2, xmm1, xmm0, 16
16107        vpclmulqdq	xmm3, xmm1, xmm0, 1
16108        vpclmulqdq	xmm4, xmm1, xmm0, 0
16109        vpclmulqdq	xmm1, xmm1, xmm0, 17
16110        vmovdqu	xmm0, OWORD PTR [rax+32]
16111        vpxor	xmm7, xmm7, xmm1
16112        vaesenc	xmm8, xmm8, xmm0
16113        vaesenc	xmm9, xmm9, xmm0
16114        vaesenc	xmm10, xmm10, xmm0
16115        vaesenc	xmm11, xmm11, xmm0
16116        vaesenc	xmm12, xmm12, xmm0
16117        vaesenc	xmm13, xmm13, xmm0
16118        vaesenc	xmm14, xmm14, xmm0
16119        vaesenc	xmm15, xmm15, xmm0
16120        ; aesenc_pclmul_n
16121        vmovdqu	xmm1, OWORD PTR [rcx+32]
16122        vmovdqu	xmm0, OWORD PTR [rsp+80]
16123        vpshufb	xmm1, xmm1, OWORD PTR L_avx2_aes_gcm_bswap_mask
16124        vpxor	xmm5, xmm5, xmm2
16125        vpclmulqdq	xmm2, xmm1, xmm0, 16
16126        vpxor	xmm5, xmm5, xmm3
16127        vpclmulqdq	xmm3, xmm1, xmm0, 1
16128        vpxor	xmm6, xmm6, xmm4
16129        vpclmulqdq	xmm4, xmm1, xmm0, 0
16130        vpclmulqdq	xmm1, xmm1, xmm0, 17
16131        vmovdqu	xmm0, OWORD PTR [rax+48]
16132        vpxor	xmm7, xmm7, xmm1
16133        vaesenc	xmm8, xmm8, xmm0
16134        vaesenc	xmm9, xmm9, xmm0
16135        vaesenc	xmm10, xmm10, xmm0
16136        vaesenc	xmm11, xmm11, xmm0
16137        vaesenc	xmm12, xmm12, xmm0
16138        vaesenc	xmm13, xmm13, xmm0
16139        vaesenc	xmm14, xmm14, xmm0
16140        vaesenc	xmm15, xmm15, xmm0
16141        ; aesenc_pclmul_n
16142        vmovdqu	xmm1, OWORD PTR [rcx+48]
16143        vmovdqu	xmm0, OWORD PTR [rsp+64]
16144        vpshufb	xmm1, xmm1, OWORD PTR L_avx2_aes_gcm_bswap_mask
16145        vpxor	xmm5, xmm5, xmm2
16146        vpclmulqdq	xmm2, xmm1, xmm0, 16
16147        vpxor	xmm5, xmm5, xmm3
16148        vpclmulqdq	xmm3, xmm1, xmm0, 1
16149        vpxor	xmm6, xmm6, xmm4
16150        vpclmulqdq	xmm4, xmm1, xmm0, 0
16151        vpclmulqdq	xmm1, xmm1, xmm0, 17
16152        vmovdqu	xmm0, OWORD PTR [rax+64]
16153        vpxor	xmm7, xmm7, xmm1
16154        vaesenc	xmm8, xmm8, xmm0
16155        vaesenc	xmm9, xmm9, xmm0
16156        vaesenc	xmm10, xmm10, xmm0
16157        vaesenc	xmm11, xmm11, xmm0
16158        vaesenc	xmm12, xmm12, xmm0
16159        vaesenc	xmm13, xmm13, xmm0
16160        vaesenc	xmm14, xmm14, xmm0
16161        vaesenc	xmm15, xmm15, xmm0
16162        ; aesenc_pclmul_n
16163        vmovdqu	xmm1, OWORD PTR [rcx+64]
16164        vmovdqu	xmm0, OWORD PTR [rsp+48]
16165        vpshufb	xmm1, xmm1, OWORD PTR L_avx2_aes_gcm_bswap_mask
16166        vpxor	xmm5, xmm5, xmm2
16167        vpclmulqdq	xmm2, xmm1, xmm0, 16
16168        vpxor	xmm5, xmm5, xmm3
16169        vpclmulqdq	xmm3, xmm1, xmm0, 1
16170        vpxor	xmm6, xmm6, xmm4
16171        vpclmulqdq	xmm4, xmm1, xmm0, 0
16172        vpclmulqdq	xmm1, xmm1, xmm0, 17
16173        vmovdqu	xmm0, OWORD PTR [rax+80]
16174        vpxor	xmm7, xmm7, xmm1
16175        vaesenc	xmm8, xmm8, xmm0
16176        vaesenc	xmm9, xmm9, xmm0
16177        vaesenc	xmm10, xmm10, xmm0
16178        vaesenc	xmm11, xmm11, xmm0
16179        vaesenc	xmm12, xmm12, xmm0
16180        vaesenc	xmm13, xmm13, xmm0
16181        vaesenc	xmm14, xmm14, xmm0
16182        vaesenc	xmm15, xmm15, xmm0
16183        ; aesenc_pclmul_n
16184        vmovdqu	xmm1, OWORD PTR [rcx+80]
16185        vmovdqu	xmm0, OWORD PTR [rsp+32]
16186        vpshufb	xmm1, xmm1, OWORD PTR L_avx2_aes_gcm_bswap_mask
16187        vpxor	xmm5, xmm5, xmm2
16188        vpclmulqdq	xmm2, xmm1, xmm0, 16
16189        vpxor	xmm5, xmm5, xmm3
16190        vpclmulqdq	xmm3, xmm1, xmm0, 1
16191        vpxor	xmm6, xmm6, xmm4
16192        vpclmulqdq	xmm4, xmm1, xmm0, 0
16193        vpclmulqdq	xmm1, xmm1, xmm0, 17
16194        vmovdqu	xmm0, OWORD PTR [rax+96]
16195        vpxor	xmm7, xmm7, xmm1
16196        vaesenc	xmm8, xmm8, xmm0
16197        vaesenc	xmm9, xmm9, xmm0
16198        vaesenc	xmm10, xmm10, xmm0
16199        vaesenc	xmm11, xmm11, xmm0
16200        vaesenc	xmm12, xmm12, xmm0
16201        vaesenc	xmm13, xmm13, xmm0
16202        vaesenc	xmm14, xmm14, xmm0
16203        vaesenc	xmm15, xmm15, xmm0
16204        ; aesenc_pclmul_n
16205        vmovdqu	xmm1, OWORD PTR [rcx+96]
16206        vmovdqu	xmm0, OWORD PTR [rsp+16]
16207        vpshufb	xmm1, xmm1, OWORD PTR L_avx2_aes_gcm_bswap_mask
16208        vpxor	xmm5, xmm5, xmm2
16209        vpclmulqdq	xmm2, xmm1, xmm0, 16
16210        vpxor	xmm5, xmm5, xmm3
16211        vpclmulqdq	xmm3, xmm1, xmm0, 1
16212        vpxor	xmm6, xmm6, xmm4
16213        vpclmulqdq	xmm4, xmm1, xmm0, 0
16214        vpclmulqdq	xmm1, xmm1, xmm0, 17
16215        vmovdqu	xmm0, OWORD PTR [rax+112]
16216        vpxor	xmm7, xmm7, xmm1
16217        vaesenc	xmm8, xmm8, xmm0
16218        vaesenc	xmm9, xmm9, xmm0
16219        vaesenc	xmm10, xmm10, xmm0
16220        vaesenc	xmm11, xmm11, xmm0
16221        vaesenc	xmm12, xmm12, xmm0
16222        vaesenc	xmm13, xmm13, xmm0
16223        vaesenc	xmm14, xmm14, xmm0
16224        vaesenc	xmm15, xmm15, xmm0
16225        ; aesenc_pclmul_n
16226        vmovdqu	xmm1, OWORD PTR [rcx+112]
16227        vmovdqu	xmm0, OWORD PTR [rsp]
16228        vpshufb	xmm1, xmm1, OWORD PTR L_avx2_aes_gcm_bswap_mask
16229        vpxor	xmm5, xmm5, xmm2
16230        vpclmulqdq	xmm2, xmm1, xmm0, 16
16231        vpxor	xmm5, xmm5, xmm3
16232        vpclmulqdq	xmm3, xmm1, xmm0, 1
16233        vpxor	xmm6, xmm6, xmm4
16234        vpclmulqdq	xmm4, xmm1, xmm0, 0
16235        vpclmulqdq	xmm1, xmm1, xmm0, 17
16236        vmovdqu	xmm0, OWORD PTR [rax+128]
16237        vpxor	xmm7, xmm7, xmm1
16238        vaesenc	xmm8, xmm8, xmm0
16239        vaesenc	xmm9, xmm9, xmm0
16240        vaesenc	xmm10, xmm10, xmm0
16241        vaesenc	xmm11, xmm11, xmm0
16242        vaesenc	xmm12, xmm12, xmm0
16243        vaesenc	xmm13, xmm13, xmm0
16244        vaesenc	xmm14, xmm14, xmm0
16245        vaesenc	xmm15, xmm15, xmm0
16246        ; aesenc_pclmul_l
16247        vpxor	xmm5, xmm5, xmm2
16248        vpxor	xmm6, xmm6, xmm4
16249        vpxor	xmm5, xmm5, xmm3
16250        vpslldq	xmm1, xmm5, 8
16251        vpsrldq	xmm5, xmm5, 8
16252        vmovdqu	xmm4, OWORD PTR [rax+144]
16253        vmovdqu	xmm0, OWORD PTR L_avx2_aes_gcm_mod2_128
16254        vaesenc	xmm8, xmm8, xmm4
16255        vpxor	xmm6, xmm6, xmm1
16256        vpxor	xmm7, xmm7, xmm5
16257        vpclmulqdq	xmm3, xmm6, xmm0, 16
16258        vaesenc	xmm9, xmm9, xmm4
16259        vaesenc	xmm10, xmm10, xmm4
16260        vaesenc	xmm11, xmm11, xmm4
16261        vpshufd	xmm6, xmm6, 78
16262        vpxor	xmm6, xmm6, xmm3
16263        vpclmulqdq	xmm3, xmm6, xmm0, 16
16264        vaesenc	xmm12, xmm12, xmm4
16265        vaesenc	xmm13, xmm13, xmm4
16266        vaesenc	xmm14, xmm14, xmm4
16267        vpshufd	xmm6, xmm6, 78
16268        vpxor	xmm6, xmm6, xmm3
16269        vpxor	xmm6, xmm6, xmm7
16270        vaesenc	xmm15, xmm15, xmm4
16271        cmp	r8d, 11
16272        vmovdqu	xmm7, OWORD PTR [rax+160]
16273        jl	L_AES_GCM_decrypt_update_avx2_aesenc_128_ghash_avx_done
16274        vaesenc	xmm8, xmm8, xmm7
16275        vaesenc	xmm9, xmm9, xmm7
16276        vaesenc	xmm10, xmm10, xmm7
16277        vaesenc	xmm11, xmm11, xmm7
16278        vaesenc	xmm12, xmm12, xmm7
16279        vaesenc	xmm13, xmm13, xmm7
16280        vaesenc	xmm14, xmm14, xmm7
16281        vaesenc	xmm15, xmm15, xmm7
16282        vmovdqu	xmm7, OWORD PTR [rax+176]
16283        vaesenc	xmm8, xmm8, xmm7
16284        vaesenc	xmm9, xmm9, xmm7
16285        vaesenc	xmm10, xmm10, xmm7
16286        vaesenc	xmm11, xmm11, xmm7
16287        vaesenc	xmm12, xmm12, xmm7
16288        vaesenc	xmm13, xmm13, xmm7
16289        vaesenc	xmm14, xmm14, xmm7
16290        vaesenc	xmm15, xmm15, xmm7
16291        cmp	r8d, 13
16292        vmovdqu	xmm7, OWORD PTR [rax+192]
16293        jl	L_AES_GCM_decrypt_update_avx2_aesenc_128_ghash_avx_done
16294        vaesenc	xmm8, xmm8, xmm7
16295        vaesenc	xmm9, xmm9, xmm7
16296        vaesenc	xmm10, xmm10, xmm7
16297        vaesenc	xmm11, xmm11, xmm7
16298        vaesenc	xmm12, xmm12, xmm7
16299        vaesenc	xmm13, xmm13, xmm7
16300        vaesenc	xmm14, xmm14, xmm7
16301        vaesenc	xmm15, xmm15, xmm7
16302        vmovdqu	xmm7, OWORD PTR [rax+208]
16303        vaesenc	xmm8, xmm8, xmm7
16304        vaesenc	xmm9, xmm9, xmm7
16305        vaesenc	xmm10, xmm10, xmm7
16306        vaesenc	xmm11, xmm11, xmm7
16307        vaesenc	xmm12, xmm12, xmm7
16308        vaesenc	xmm13, xmm13, xmm7
16309        vaesenc	xmm14, xmm14, xmm7
16310        vaesenc	xmm15, xmm15, xmm7
16311        vmovdqu	xmm7, OWORD PTR [rax+224]
16312L_AES_GCM_decrypt_update_avx2_aesenc_128_ghash_avx_done:
16313        ; aesenc_last
16314        vaesenclast	xmm8, xmm8, xmm7
16315        vaesenclast	xmm9, xmm9, xmm7
16316        vaesenclast	xmm10, xmm10, xmm7
16317        vaesenclast	xmm11, xmm11, xmm7
16318        vmovdqu	xmm0, OWORD PTR [rcx]
16319        vmovdqu	xmm1, OWORD PTR [rcx+16]
16320        vmovdqu	xmm2, OWORD PTR [rcx+32]
16321        vmovdqu	xmm3, OWORD PTR [rcx+48]
16322        vpxor	xmm8, xmm8, xmm0
16323        vpxor	xmm9, xmm9, xmm1
16324        vpxor	xmm10, xmm10, xmm2
16325        vpxor	xmm11, xmm11, xmm3
16326        vmovdqu	OWORD PTR [rdx], xmm8
16327        vmovdqu	OWORD PTR [rdx+16], xmm9
16328        vmovdqu	OWORD PTR [rdx+32], xmm10
16329        vmovdqu	OWORD PTR [rdx+48], xmm11
16330        vaesenclast	xmm12, xmm12, xmm7
16331        vaesenclast	xmm13, xmm13, xmm7
16332        vaesenclast	xmm14, xmm14, xmm7
16333        vaesenclast	xmm15, xmm15, xmm7
16334        vmovdqu	xmm0, OWORD PTR [rcx+64]
16335        vmovdqu	xmm1, OWORD PTR [rcx+80]
16336        vmovdqu	xmm2, OWORD PTR [rcx+96]
16337        vmovdqu	xmm3, OWORD PTR [rcx+112]
16338        vpxor	xmm12, xmm12, xmm0
16339        vpxor	xmm13, xmm13, xmm1
16340        vpxor	xmm14, xmm14, xmm2
16341        vpxor	xmm15, xmm15, xmm3
16342        vmovdqu	OWORD PTR [rdx+64], xmm12
16343        vmovdqu	OWORD PTR [rdx+80], xmm13
16344        vmovdqu	OWORD PTR [rdx+96], xmm14
16345        vmovdqu	OWORD PTR [rdx+112], xmm15
16346        ; aesenc_128_ghash - end
16347        add	edi, 128
16348        cmp	edi, r13d
16349        jl	L_AES_GCM_decrypt_update_avx2_ghash_128
16350        vmovdqu	xmm5, OWORD PTR [rsp]
16351        vmovdqu	xmm4, OWORD PTR [rsp+128]
16352        vmovdqu	xmm15, OWORD PTR [rsp+144]
16353L_AES_GCM_decrypt_update_avx2_done_128:
16354        cmp	edi, r9d
16355        jge	L_AES_GCM_decrypt_update_avx2_done_dec
16356        mov	r13d, r9d
16357        and	r13d, 4294967280
16358        cmp	edi, r13d
16359        jge	L_AES_GCM_decrypt_update_avx2_last_block_done
16360L_AES_GCM_decrypt_update_avx2_last_block_start:
16361        vmovdqu	xmm11, OWORD PTR [r11+rdi]
16362        vpshufb	xmm10, xmm4, OWORD PTR L_avx2_aes_gcm_bswap_epi64
16363        vpshufb	xmm12, xmm11, OWORD PTR L_avx2_aes_gcm_bswap_mask
16364        vpaddd	xmm4, xmm4, OWORD PTR L_avx2_aes_gcm_one
16365        vpxor	xmm12, xmm12, xmm6
16366        ; aesenc_gfmul_sb
16367        vpclmulqdq	xmm2, xmm12, xmm5, 1
16368        vpclmulqdq	xmm3, xmm12, xmm5, 16
16369        vpclmulqdq	xmm1, xmm12, xmm5, 0
16370        vpclmulqdq	xmm8, xmm12, xmm5, 17
16371        vpxor	xmm10, xmm10, [rax]
16372        vaesenc	xmm10, xmm10, [rax+16]
16373        vpxor	xmm3, xmm3, xmm2
16374        vpslldq	xmm2, xmm3, 8
16375        vpsrldq	xmm3, xmm3, 8
16376        vaesenc	xmm10, xmm10, [rax+32]
16377        vpxor	xmm2, xmm2, xmm1
16378        vpclmulqdq	xmm1, xmm2, OWORD PTR L_avx2_aes_gcm_mod2_128, 16
16379        vaesenc	xmm10, xmm10, [rax+48]
16380        vaesenc	xmm10, xmm10, [rax+64]
16381        vaesenc	xmm10, xmm10, [rax+80]
16382        vpshufd	xmm2, xmm2, 78
16383        vpxor	xmm2, xmm2, xmm1
16384        vpclmulqdq	xmm1, xmm2, OWORD PTR L_avx2_aes_gcm_mod2_128, 16
16385        vaesenc	xmm10, xmm10, [rax+96]
16386        vaesenc	xmm10, xmm10, [rax+112]
16387        vaesenc	xmm10, xmm10, [rax+128]
16388        vpshufd	xmm2, xmm2, 78
16389        vaesenc	xmm10, xmm10, [rax+144]
16390        vpxor	xmm8, xmm8, xmm3
16391        vpxor	xmm2, xmm2, xmm8
16392        vmovdqu	xmm0, OWORD PTR [rax+160]
16393        cmp	r8d, 11
16394        jl	L_AES_GCM_decrypt_update_avx2_aesenc_gfmul_sb_last
16395        vaesenc	xmm10, xmm10, xmm0
16396        vaesenc	xmm10, xmm10, [rax+176]
16397        vmovdqu	xmm0, OWORD PTR [rax+192]
16398        cmp	r8d, 13
16399        jl	L_AES_GCM_decrypt_update_avx2_aesenc_gfmul_sb_last
16400        vaesenc	xmm10, xmm10, xmm0
16401        vaesenc	xmm10, xmm10, [rax+208]
16402        vmovdqu	xmm0, OWORD PTR [rax+224]
16403L_AES_GCM_decrypt_update_avx2_aesenc_gfmul_sb_last:
16404        vaesenclast	xmm10, xmm10, xmm0
16405        vpxor	xmm6, xmm2, xmm1
16406        vpxor	xmm10, xmm10, xmm11
16407        vmovdqu	OWORD PTR [r10+rdi], xmm10
16408        add	edi, 16
16409        cmp	edi, r13d
16410        jl	L_AES_GCM_decrypt_update_avx2_last_block_start
16411L_AES_GCM_decrypt_update_avx2_last_block_done:
16412L_AES_GCM_decrypt_update_avx2_done_dec:
16413        vmovdqu	OWORD PTR [r12], xmm6
16414        vmovdqu	OWORD PTR [r15], xmm4
16415        vzeroupper
16416        vmovdqu	xmm6, OWORD PTR [rsp+168]
16417        vmovdqu	xmm7, OWORD PTR [rsp+184]
16418        vmovdqu	xmm8, OWORD PTR [rsp+200]
16419        vmovdqu	xmm9, OWORD PTR [rsp+216]
16420        vmovdqu	xmm10, OWORD PTR [rsp+232]
16421        vmovdqu	xmm11, OWORD PTR [rsp+248]
16422        vmovdqu	xmm12, OWORD PTR [rsp+264]
16423        vmovdqu	xmm13, OWORD PTR [rsp+280]
16424        vmovdqu	xmm14, OWORD PTR [rsp+296]
16425        vmovdqu	xmm15, OWORD PTR [rsp+312]
16426        add	rsp, 328
16427        pop	rdi
16428        pop	r15
16429        pop	r14
16430        pop	r12
16431        pop	r13
16432        ret
16433AES_GCM_decrypt_update_avx2 ENDP
16434_TEXT ENDS
16435_TEXT SEGMENT READONLY PARA
16436AES_GCM_decrypt_final_avx2 PROC
16437        push	r12
16438        push	r13
16439        push	r14
16440        mov	eax, DWORD PTR [rsp+64]
16441        mov	r10, QWORD PTR [rsp+72]
16442        mov	r11, QWORD PTR [rsp+80]
16443        mov	r12, QWORD PTR [rsp+88]
16444        sub	rsp, 48
16445        vmovdqu	OWORD PTR [rsp+16], xmm6
16446        vmovdqu	OWORD PTR [rsp+32], xmm7
16447        vmovdqu	xmm4, OWORD PTR [rcx]
16448        vmovdqu	xmm5, OWORD PTR [r10]
16449        vmovdqu	xmm6, OWORD PTR [r11]
16450        vpsrlq	xmm1, xmm5, 63
16451        vpsllq	xmm0, xmm5, 1
16452        vpslldq	xmm1, xmm1, 8
16453        vpor	xmm0, xmm0, xmm1
16454        vpshufd	xmm5, xmm5, 255
16455        vpsrad	xmm5, xmm5, 31
16456        vpand	xmm5, xmm5, OWORD PTR L_avx2_aes_gcm_mod2_128
16457        vpxor	xmm5, xmm5, xmm0
16458        ; calc_tag
16459        shl	r9, 3
16460        shl	rax, 3
16461        vmovq	xmm0, r9
16462        vmovq	xmm1, rax
16463        vpunpcklqdq	xmm0, xmm0, xmm1
16464        vpxor	xmm0, xmm0, xmm4
16465        ; ghash_gfmul_red
16466        vpclmulqdq	xmm7, xmm0, xmm5, 16
16467        vpclmulqdq	xmm3, xmm0, xmm5, 1
16468        vpclmulqdq	xmm2, xmm0, xmm5, 0
16469        vpxor	xmm7, xmm7, xmm3
16470        vpslldq	xmm3, xmm7, 8
16471        vpsrldq	xmm7, xmm7, 8
16472        vpxor	xmm3, xmm3, xmm2
16473        vpclmulqdq	xmm0, xmm0, xmm5, 17
16474        vpclmulqdq	xmm2, xmm3, OWORD PTR L_avx2_aes_gcm_mod2_128, 16
16475        vpshufd	xmm3, xmm3, 78
16476        vpxor	xmm3, xmm3, xmm2
16477        vpclmulqdq	xmm2, xmm3, OWORD PTR L_avx2_aes_gcm_mod2_128, 16
16478        vpshufd	xmm3, xmm3, 78
16479        vpxor	xmm0, xmm0, xmm7
16480        vpxor	xmm0, xmm0, xmm3
16481        vpxor	xmm0, xmm0, xmm2
16482        vpshufb	xmm0, xmm0, OWORD PTR L_avx2_aes_gcm_bswap_mask
16483        vpxor	xmm0, xmm0, xmm6
16484        ; cmp_tag
16485        cmp	r8d, 16
16486        je	L_AES_GCM_decrypt_final_avx2_cmp_tag_16
16487        xor	r13, r13
16488        xor	r10, r10
16489        vmovdqu	OWORD PTR [rsp], xmm0
16490L_AES_GCM_decrypt_final_avx2_cmp_tag_loop:
16491        movzx	r14d, BYTE PTR [rsp+r13]
16492        xor	r14b, BYTE PTR [rdx+r13]
16493        or	r10b, r14b
16494        inc	r13d
16495        cmp	r13d, r8d
16496        jne	L_AES_GCM_decrypt_final_avx2_cmp_tag_loop
16497        cmp	r10b, 0
16498        sete	r10b
16499        jmp	L_AES_GCM_decrypt_final_avx2_cmp_tag_done
16500L_AES_GCM_decrypt_final_avx2_cmp_tag_16:
16501        vmovdqu	xmm1, OWORD PTR [rdx]
16502        vpcmpeqb	xmm0, xmm0, xmm1
16503        vpmovmskb	r13, xmm0
16504        ; %%edx == 0xFFFF then return 1 else => return 0
16505        xor	r10d, r10d
16506        cmp	r13d, 65535
16507        sete	r10b
16508L_AES_GCM_decrypt_final_avx2_cmp_tag_done:
16509        mov	DWORD PTR [r12], r10d
16510        vzeroupper
16511        vmovdqu	xmm6, OWORD PTR [rsp+16]
16512        vmovdqu	xmm7, OWORD PTR [rsp+32]
16513        add	rsp, 48
16514        pop	r14
16515        pop	r13
16516        pop	r12
16517        ret
16518AES_GCM_decrypt_final_avx2 ENDP
16519_TEXT ENDS
16520ENDIF
16521END