start_time_stamp,end_time_stamp,device_addr,stream_id,pool_type,size,actual_peak_memory,file_name,line_num,type,producer_task,task_name,node_name,graph_name,user_tasks,last_user_task,python_stack,is_persistent,is_small, 2,9223372036854775807,0x12c180000000,0,DefaultEnhancedAscendMemoryPool,512,2097152,mindspore/ccsrc/pyboost/pyboost_utils.cc,332,PyNativeInput,0,PyNative,Reshape,,{0},0,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:96;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:80;Function:build_model|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/version_control.py;Line:121;Function:decorator|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/models/llama/llama.py;Line:484;Function:__init__|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/models/modeling_utils.py;Line:278;Function:__init__|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/generation/text_generator.py;Line:98;Function:__init__|,0,0, 4,9223372036854775807,0x12c180000200,0,DefaultEnhancedAscendMemoryPool,512,2097152,mindspore/ccsrc/pyboost/pyboost_utils.cc,332,PyNativeInput,2,PyNative,Reshape,,{2},2,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:96;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:80;Function:build_model|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/version_control.py;Line:121;Function:decorator|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/models/llama/llama.py;Line:511;Function:__init__|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/models/llama/llama.py;Line:81;Function:__init__|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/models/modeling_utils.py;Line:278;Function:__init__|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/generation/text_generator.py;Line:98;Function:__init__|,0,0, 6,9223372036854775807,0x12c180000400,0,DefaultEnhancedAscendMemoryPool,512,2097152,mindspore/ccsrc/pyboost/pyboost_utils.cc,332,PyNativeInput,4,PyNative,Reshape,,{4},4,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:96;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:80;Function:build_model|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/version_control.py;Line:121;Function:decorator|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/models/llama/llama.py;Line:511;Function:__init__|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/models/llama/llama.py;Line:161;Function:__init__|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/tools/logger.py;Line:606;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/modules/transformer/transformer.py;Line:1264;Function:__init__|,0,0, 8,9223372036854775807,0x12c580000000,0,DefaultEnhancedAscendMemoryPool,512,4194304,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,6,PrepareDataForSequenceAndScalarValue,Default/data-599,,{386-427-533-574-703-767-782-916-930-945-1013-1028-1160-1174-1189-2671-2712-2818-2859-2988-3052-3067-3201-3215-3230-3298-3313-3445-3459-3474-4961-5002-5108-5149-5278-5342-5357-5491-5505-5520-5588-5603-5735-5749-5764-7251-7292-7398-7439-7568-7632-7647-7781-7795-7810-7878-7893-8025-8039-8054-9541-9582-9688-9729-9858-9922-9937-10071-10085-10100-10168-10183-10315-10329-10344-11831-11872-11978-12019-12148-12212-12227-12361-12375-12390-12458-12473-12605-12619-12634-14121-14162-14268-14309-14438-14502-14517-14651-14665-14680-14748-14763-14895-14909-14924-16411-16452-16558-16599-16728-16792-16807-16941-16955-16970-17038-17053-17185-17199-17214-18701-18742-18848-18889-19018-19082-19097-19231-19245-19260-19328-19343-19475-19489-19504-20991-21032-21138-21179-21308-21372-21387-21521-21535-21550-21618-21633-21765-21779-21794},21794,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 10,9223372036854775807,0x12c580000200,0,DefaultEnhancedAscendMemoryPool,512,4194304,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,8,SyncTensorData,Default/data-600,,{1831-4116-6406-8696-10986-13276-15566-17856-20146-22436},22436,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 12,9223372036854775807,0x12c580000400,0,DefaultEnhancedAscendMemoryPool,512,4194304,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,10,SyncTensorData,Default/data-601,,{1822-4107-6397-8687-10977-13267-15557-17847-20137-22427},22427,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 14,9223372036854775807,0x12c580000600,0,DefaultEnhancedAscendMemoryPool,512,4194304,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,12,PrepareDataForSequenceAndScalarValue,Default/data-602,,{},,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 16,9223372036854775807,0x12c580000800,0,DefaultEnhancedAscendMemoryPool,512,4194304,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,14,PrepareDataForSequenceAndScalarValue,Default/data-603,,{270-270-297-312-323-323-335-346-346-384-406-413-425-447-462-476-476-491-491-507-531-553-560-572-594-612-623-668-668-679-679-691-699-699-729-737-737-756-763-763-770-778-778-816-824-824-867-888-905-912-912-919-926-926-933-941-941-975-983-983-1002-1009-1009-1016-1024-1024-1057-1065-1065-1109-1132-1149-1156-1156-1163-1170-1170-1177-1185-1185-1296-1304-1312-1320-1328-1336-1344-1352-1360-1368-1376-1384-1392-1400-1408-1416-1424-1432-1440-1448-1458-2555-2555-2582-2597-2608-2608-2620-2631-2631-2669-2691-2698-2710-2732-2747-2761-2761-2776-2776-2792-2816-2838-2845-2857-2879-2897-2908-2953-2953-2964-2964-2976-2984-2984-3014-3022-3022-3041-3048-3048-3055-3063-3063-3101-3109-3109-3152-3173-3190-3197-3197-3204-3211-3211-3218-3226-3226-3260-3268-3268-3287-3294-3294-3301-3309-3309-3342-3350-3350-3394-3417-3434-3441-3441-3448-3455-3455-3462-3470-3470-3581-3589-3597-3605-3613-3621-3629-3637-3645-3653-3661-3669-3677-3685-3693-3701-3709-3717-3725-3733-3743-4845-4845-4872-4887-4898-4898-4910-4921-4921-4959-4981-4988-5000-5022-5037-5051-5051-5066-5066-5082-5106-5128-5135-5147-5169-5187-5198-5243-5243-5254-5254-5266-5274-5274-5304-5312-5312-5331-5338-5338-5345-5353-5353-5391-5399-5399-5442-5463-5480-5487-5487-5494-5501-5501-5508-5516-5516-5550-5558-5558-5577-5584-5584-5591-5599-5599-5632-5640-5640-5684-5707-5724-5731-5731-5738-5745-5745-5752-5760-5760-5871-5879-5887-5895-5903-5911-5919-5927-5935-5943-5951-5959-5967-5975-5983-5991-5999-6007-6015-6023-6033-7135-7135-7162-7177-7188-7188-7200-7211-7211-7249-7271-7278-7290-7312-7327-7341-7341-7356-7356-7372-7396-7418-7425-7437-7459-7477-7488-7533-7533-7544-7544-7556-7564-7564-7594-7602-7602-7621-7628-7628-7635-7643-7643-7681-7689-7689-7732-7753-7770-7777-7777-7784-7791-7791-7798-7806-7806-7840-7848-7848-7867-7874-7874-7881-7889-7889-7922-7930-7930-7974-7997-8014-8021-8021-8028-8035-8035-8042-8050-8050-8161-8169-8177-8185-8193-8201-8209-8217-8225-8233-8241-8249-8257-8265-8273-8281-8289-8297-8305-8313-8323-9425-9425-9452-9467-9478-9478-9490-9501-9501-9539-9561-9568-9580-9602-9617-9631-9631-9646-9646-9662-9686-9708-9715-9727-9749-9767-9778-9823-9823-9834-9834-9846-9854-9854-9884-9892-9892-9911-9918-9918-9925-9933-9933-9971-9979-9979-10022-10043-10060-10067-10067-10074-10081-10081-10088-10096-10096-10130-10138-10138-10157-10164-10164-10171-10179-10179-10212-10220-10220-10264-10287-10304-10311-10311-10318-10325-10325-10332-10340-10340-10451-10459-10467-10475-10483-10491-10499-10507-10515-10523-10531-10539-10547-10555-10563-10571-10579-10587-10595-10603-10613-11715-11715-11742-11757-11768-11768-11780-11791-11791-11829-11851-11858-11870-11892-11907-11921-11921-11936-11936-11952-11976-11998-12005-12017-12039-12057-12068-12113-12113-12124-12124-12136-12144-12144-12174-12182-12182-12201-12208-12208-12215-12223-12223-12261-12269-12269-12312-12333-12350-12357-12357-12364-12371-12371-12378-12386-12386-12420-12428-12428-12447-12454-12454-12461-12469-12469-12502-12510-12510-12554-12577-12594-12601-12601-12608-12615-12615-12622-12630-12630-12741-12749-12757-12765-12773-12781-12789-12797-12805-12813-12821-12829-12837-12845-12853-12861-12869-12877-12885-12893-12903-14005-14005-14032-14047-14058-14058-14070-14081-14081-14119-14141-14148-14160-14182-14197-14211-14211-14226-14226-14242-14266-14288-14295-14307-14329-14347-14358-14403-14403-14414-14414-14426-14434-14434-14464-14472-14472-14491-14498-14498-14505-14513-14513-14551-14559-14559-14602-14623-14640-14647-14647-14654-14661-14661-14668-14676-14676-14710-14718-14718-14737-14744-14744-14751-14759-14759-14792-14800-14800-14844-14867-14884-14891-14891-14898-14905-14905-14912-14920-14920-15031-15039-15047-15055-15063-15071-15079-15087-15095-15103-15111-15119-15127-15135-15143-15151-15159-15167-15175-15183-15193-16295-16295-16322-16337-16348-16348-16360-16371-16371-16409-16431-16438-16450-16472-16487-16501-16501-16516-16516-16532-16556-16578-16585-16597-16619-16637-16648-16693-16693-16704-16704-16716-16724-16724-16754-16762-16762-16781-16788-16788-16795-16803-16803-16841-16849-16849-16892-16913-16930-16937-16937-16944-16951-16951-16958-16966-16966-17000-17008-17008-17027-17034-17034-17041-17049-17049-17082-17090-17090-17134-17157-17174-17181-17181-17188-17195-17195-17202-17210-17210-17321-17329-17337-17345-17353-17361-17369-17377-17385-17393-17401-17409-17417-17425-17433-17441-17449-17457-17465-17473-17483-18585-18585-18612-18627-18638-18638-18650-18661-18661-18699-18721-18728-18740-18762-18777-18791-18791-18806-18806-18822-18846-18868-18875-18887-18909-18927-18938-18983-18983-18994-18994-19006-19014-19014-19044-19052-19052-19071-19078-19078-19085-19093-19093-19131-19139-19139-19182-19203-19220-19227-19227-19234-19241-19241-19248-19256-19256-19290-19298-19298-19317-19324-19324-19331-19339-19339-19372-19380-19380-19424-19447-19464-19471-19471-19478-19485-19485-19492-19500-19500-19611-19619-19627-19635-19643-19651-19659-19667-19675-19683-19691-19699-19707-19715-19723-19731-19739-19747-19755-19763-19773-20875-20875-20902-20917-20928-20928-20940-20951-20951-20989-21011-21018-21030-21052-21067-21081-21081-21096-21096-21112-21136-21158-21165-21177-21199-21217-21228-21273-21273-21284-21284-21296-21304-21304-21334-21342-21342-21361-21368-21368-21375-21383-21383-21421-21429-21429-21472-21493-21510-21517-21517-21524-21531-21531-21538-21546-21546-21580-21588-21588-21607-21614-21614-21621-21629-21629-21662-21670-21670-21714-21737-21754-21761-21761-21768-21775-21775-21782-21790-21790-21901-21909-21917-21925-21933-21941-21949-21957-21965-21973-21981-21989-21997-22005-22013-22021-22029-22037-22045-22053-22063},22063,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 18,9223372036854775807,0x12c580000a00,0,DefaultEnhancedAscendMemoryPool,512,4194304,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,16,PrepareDataForSequenceAndScalarValue,Default/data-604,,{420-567-2705-2852-4995-5142-7285-7432-9575-9722-11865-12012-14155-14302-16445-16592-18735-18882-21025-21172},21172,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 20,9223372036854775807,0x12c580000c00,0,DefaultEnhancedAscendMemoryPool,512,4194304,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,18,PrepareDataForSequenceAndScalarValue,Default/data-605,,{302-317-340-377-452-467-512-524-830-857-878-899-1071-1099-1120-1143-2587-2602-2625-2662-2737-2752-2797-2809-3115-3142-3163-3184-3356-3384-3405-3428-4877-4892-4915-4952-5027-5042-5087-5099-5405-5432-5453-5474-5646-5674-5695-5718-7167-7182-7205-7242-7317-7332-7377-7389-7695-7722-7743-7764-7936-7964-7985-8008-9457-9472-9495-9532-9607-9622-9667-9679-9985-10012-10033-10054-10226-10254-10275-10298-11747-11762-11785-11822-11897-11912-11957-11969-12275-12302-12323-12344-12516-12544-12565-12588-14037-14052-14075-14112-14187-14202-14247-14259-14565-14592-14613-14634-14806-14834-14855-14878-16327-16342-16365-16402-16477-16492-16537-16549-16855-16882-16903-16924-17096-17124-17145-17168-18617-18632-18655-18692-18767-18782-18827-18839-19145-19172-19193-19214-19386-19414-19435-19458-20907-20922-20945-20982-21057-21072-21117-21129-21435-21462-21483-21504-21676-21704-21725-21748},21748,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 22,9223372036854775807,0x12c580000e00,0,DefaultEnhancedAscendMemoryPool,512,4194304,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,20,PrepareDataForSequenceAndScalarValue,Default/data-606,,{642-2927-5217-7507-9797-12087-14377-16667-18957-21247},21247,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 24,9223372036854775807,0x12c580001000,0,DefaultEnhancedAscendMemoryPool,512,4194304,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,22,PrepareDataForSequenceAndScalarValue,Default/data-607,,{251-612-644-668-2536-2897-2929-2953-4826-5187-5219-5243-7116-7477-7509-7533-9406-9767-9799-9823-11696-12057-12089-12113-13986-14347-14379-14403-16276-16637-16669-16693-18566-18927-18959-18983-20856-21217-21249-21273},21273,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 26,9223372036854775807,0x12c580001200,0,DefaultEnhancedAscendMemoryPool,512,4194304,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,24,PrepareDataForSequenceAndScalarValue,Default/data-608,,{601-626-2886-2911-5176-5201-7466-7491-9756-9781-12046-12071-14336-14361-16626-16651-18916-18941-21206-21231},21231,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 28,9223372036854775807,0x12c580001400,0,DefaultEnhancedAscendMemoryPool,512,4194304,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,26,SyncTensorData,Default/data-609,,{},,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 30,9223372036854775807,0x12c580001600,0,DefaultEnhancedAscendMemoryPool,512,4194304,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,28,SyncTensorData,Default/data-610,,{1621-3906-6196-8486-10776-13066-15356-17646-19936-22226},22226,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 32,9223372036854775807,0x12c580001800,0,DefaultEnhancedAscendMemoryPool,512,4194304,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,30,SyncTensorData,Default/data-611,,{},,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 34,9223372036854775807,0x12c580001a00,0,DefaultEnhancedAscendMemoryPool,512,4194304,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,32,SyncTensorData,Default/data-612,,{679-2964-5254-7544-9834-12124-14414-16704-18994-21284},21284,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 36,9223372036854775807,0x12c580001c00,0,DefaultEnhancedAscendMemoryPool,512,4194304,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,34,SyncTensorData,Default/data-613,,{1816-1834-4101-4119-6391-6409-8681-8699-10971-10989-13261-13279-15551-15569-17841-17859-20131-20149-22421-22439},22439,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 38,9223372036854775807,0x12c580001e00,0,DefaultEnhancedAscendMemoryPool,512,4194304,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,36,PrepareDataForSequenceAndScalarValue,Default/data-614,,{723-810-860-881-902-972-1054-1102-1123-1146-3008-3095-3145-3166-3187-3257-3339-3387-3408-3431-5298-5385-5435-5456-5477-5547-5629-5677-5698-5721-7588-7675-7725-7746-7767-7837-7919-7967-7988-8011-9878-9965-10015-10036-10057-10127-10209-10257-10278-10301-12168-12255-12305-12326-12347-12417-12499-12547-12568-12591-14458-14545-14595-14616-14637-14707-14789-14837-14858-14881-16748-16835-16885-16906-16927-16997-17079-17127-17148-17171-19038-19125-19175-19196-19217-19287-19369-19417-19438-19461-21328-21415-21465-21486-21507-21577-21659-21707-21728-21751},21751,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 40,9223372036854775807,0x12c580002000,0,DefaultEnhancedAscendMemoryPool,512,4194304,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,38,SyncTensorData,Default/data-615,,{1618-3903-6193-8483-10773-13063-15353-17643-19933-22223},22223,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 42,9223372036854775807,0x12c580002200,0,DefaultEnhancedAscendMemoryPool,512,4194304,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,40,SyncTensorData,Default/data-616,,{},,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 44,9223372036854775807,0x12c580002400,0,DefaultEnhancedAscendMemoryPool,512,4194304,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,42,SyncTensorData,Default/data-617,,{286-2571-4861-7151-9441-11731-14021-16311-18601-20891},20891,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 46,9223372036854775807,0x12c580002600,0,DefaultEnhancedAscendMemoryPool,512,4194304,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,44,SyncTensorData,Default/data-618,,{},,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 48,9223372036854775807,0x12c580002800,0,DefaultEnhancedAscendMemoryPool,2097664,6291456,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,46,SyncTensorData,Default/data-619,,{280-2565-4855-7145-9435-11725-14015-16305-18595-20885},20885,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 50,9223372036854775807,0x12c580202a00,0,DefaultEnhancedAscendMemoryPool,512,6291456,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,48,SyncTensorData,Default/data-620,,{1501-3786-6076-8366-10656-12946-15236-17526-19816-22106},22106,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 52,9223372036854775807,0x12c580202c00,0,DefaultEnhancedAscendMemoryPool,512,6291456,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,50,SyncTensorData,Default/data-621,,{1840-4125-6415-8705-10995-13285-15575-17865-20155-22445},22445,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 54,9223372036854775807,0x12c580202e00,0,DefaultEnhancedAscendMemoryPool,512,6291456,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,52,PrepareDataForSequenceAndScalarValue,Default/data-622,,{288-373-520-838-1079-2573-2658-2805-3123-3364-4863-4948-5095-5413-5654-7153-7238-7385-7703-7944-9443-9528-9675-9993-10234-11733-11818-11965-12283-12524-14023-14108-14255-14573-14814-16313-16398-16545-16863-17104-18603-18688-18835-19153-19394-20893-20978-21125-21443-21684},21684,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 56,9223372036854775807,0x12c580203000,0,DefaultEnhancedAscendMemoryPool,512,6291456,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,54,SyncTensorData,Default/data-623,,{1861-1893-1925-1957-1989-2021-2053-2085-2117-2149-2181-2213-2245-2277-2309-2341-2373-2405-2437-2469-2501-4146-4178-4210-4242-4274-4306-4338-4370-4402-4434-4466-4498-4530-4562-4594-4626-4658-4690-4722-4754-4786-6436-6468-6500-6532-6564-6596-6628-6660-6692-6724-6756-6788-6820-6852-6884-6916-6948-6980-7012-7044-7076-8726-8758-8790-8822-8854-8886-8918-8950-8982-9014-9046-9078-9110-9142-9174-9206-9238-9270-9302-9334-9366-11016-11048-11080-11112-11144-11176-11208-11240-11272-11304-11336-11368-11400-11432-11464-11496-11528-11560-11592-11624-11656-13306-13338-13370-13402-13434-13466-13498-13530-13562-13594-13626-13658-13690-13722-13754-13786-13818-13850-13882-13914-13946-15596-15628-15660-15692-15724-15756-15788-15820-15852-15884-15916-15948-15980-16012-16044-16076-16108-16140-16172-16204-16236-17886-17918-17950-17982-18014-18046-18078-18110-18142-18174-18206-18238-18270-18302-18334-18366-18398-18430-18462-18494-18526-20176-20208-20240-20272-20304-20336-20368-20400-20432-20464-20496-20528-20560-20592-20624-20656-20688-20720-20752-20784-20816-22466-22498-22530-22562-22594-22626-22658-22690-22722-22754-22786-22818-22850-22882-22914-22946-22978-23010-23042-23074-23106},23106,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 58,9223372036854775807,0x12c580203200,0,DefaultEnhancedAscendMemoryPool,512,6291456,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,56,PrepareDataForSequenceAndScalarValue,Default/data-624,,{},,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 60,9223372036854775807,0x12c580203400,0,DefaultEnhancedAscendMemoryPool,512,6291456,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,58,PrepareDataForSequenceAndScalarValue,Default/data-625,,{256-329-2541-2614-4831-4904-7121-7194-9411-9484-11701-11774-13991-14064-16281-16354-18571-18644-20861-20934},20934,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 62,9223372036854775807,0x12c580203600,0,DefaultEnhancedAscendMemoryPool,512,6291456,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,60,PrepareDataForSequenceAndScalarValue,Default/data-626,,{278-2563-4853-7143-9433-11723-14013-16303-18593-20883},20883,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 64,9223372036854775807,0x12c580203800,0,DefaultEnhancedAscendMemoryPool,512,6291456,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,62,SyncTensorData,Default/data-627,,{1828-4113-6403-8693-10983-13273-15563-17853-20143-22433},22433,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 66,9223372036854775807,0x12c580203a00,0,DefaultEnhancedAscendMemoryPool,512,6291456,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,64,PrepareDataForSequenceAndScalarValue,Default/data-628,,{263-395-436-542-583-2548-2680-2721-2827-2868-4838-4970-5011-5117-5158-7128-7260-7301-7407-7448-9418-9550-9591-9697-9738-11708-11840-11881-11987-12028-13998-14130-14171-14277-14318-16288-16420-16461-16567-16608-18578-18710-18751-18857-18898-20868-21000-21041-21147-21188},21188,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 68,9223372036854775807,0x12c580203c00,0,DefaultEnhancedAscendMemoryPool,512,6291456,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,66,PrepareDataForSequenceAndScalarValue,Default/data-629,,{628-2913-5203-7493-9783-12073-14363-16653-18943-21233},21233,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 70,9223372036854775807,0x12c580203e00,0,DefaultEnhancedAscendMemoryPool,512,6291456,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,68,SyncTensorData,Default/data-630,,{1609-3894-6184-8474-10764-13054-15344-17634-19924-22214},22214,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 72,9223372036854775807,0x12c580204000,0,DefaultEnhancedAscendMemoryPool,512,6291456,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,70,PrepareDataForSequenceAndScalarValue,Default/data-631,,{373-520-838-1079-2658-2805-3123-3364-4948-5095-5413-5654-7238-7385-7703-7944-9528-9675-9993-10234-11818-11965-12283-12524-14108-14255-14573-14814-16398-16545-16863-17104-18688-18835-19153-19394-20978-21125-21443-21684},21684,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 74,9223372036854775807,0x12c580204200,0,DefaultEnhancedAscendMemoryPool,512,6291456,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,72,SyncTensorData,Default/data-632,,{1627-3912-6202-8492-10782-13072-15362-17652-19942-22232},22232,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 76,9223372036854775807,0x12c580204400,0,DefaultEnhancedAscendMemoryPool,512,6291456,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,74,SyncTensorData,Default/data-633,,{1642-1650-1658-1666-1674-1682-1690-1698-1706-1714-1722-1730-1738-1746-1754-1762-1770-1778-1786-1796-1804-3927-3935-3943-3951-3959-3967-3975-3983-3991-3999-4007-4015-4023-4031-4039-4047-4055-4063-4071-4081-4089-6217-6225-6233-6241-6249-6257-6265-6273-6281-6289-6297-6305-6313-6321-6329-6337-6345-6353-6361-6371-6379-8507-8515-8523-8531-8539-8547-8555-8563-8571-8579-8587-8595-8603-8611-8619-8627-8635-8643-8651-8661-8669-10797-10805-10813-10821-10829-10837-10845-10853-10861-10869-10877-10885-10893-10901-10909-10917-10925-10933-10941-10951-10959-13087-13095-13103-13111-13119-13127-13135-13143-13151-13159-13167-13175-13183-13191-13199-13207-13215-13223-13231-13241-13249-15377-15385-15393-15401-15409-15417-15425-15433-15441-15449-15457-15465-15473-15481-15489-15497-15505-15513-15521-15531-15539-17667-17675-17683-17691-17699-17707-17715-17723-17731-17739-17747-17755-17763-17771-17779-17787-17795-17803-17811-17821-17829-19957-19965-19973-19981-19989-19997-20005-20013-20021-20029-20037-20045-20053-20061-20069-20077-20085-20093-20101-20111-20119-22247-22255-22263-22271-22279-22287-22295-22303-22311-22319-22327-22335-22343-22351-22359-22367-22375-22383-22391-22401-22409},22409,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 78,9223372036854775807,0x12c580204600,0,DefaultEnhancedAscendMemoryPool,512,6291456,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,76,SyncTensorData,Default/data-634,,{1849-1881-1913-1945-1977-2009-2041-2073-2105-2137-2169-2201-2233-2265-2297-2329-2361-2393-2425-2455-2487-4134-4166-4198-4230-4262-4294-4326-4358-4390-4422-4454-4486-4518-4550-4582-4614-4646-4678-4710-4740-4772-6424-6456-6488-6520-6552-6584-6616-6648-6680-6712-6744-6776-6808-6840-6872-6904-6936-6968-7000-7030-7062-8714-8746-8778-8810-8842-8874-8906-8938-8970-9002-9034-9066-9098-9130-9162-9194-9226-9258-9290-9320-9352-11004-11036-11068-11100-11132-11164-11196-11228-11260-11292-11324-11356-11388-11420-11452-11484-11516-11548-11580-11610-11642-13294-13326-13358-13390-13422-13454-13486-13518-13550-13582-13614-13646-13678-13710-13742-13774-13806-13838-13870-13900-13932-15584-15616-15648-15680-15712-15744-15776-15808-15840-15872-15904-15936-15968-16000-16032-16064-16096-16128-16160-16190-16222-17874-17906-17938-17970-18002-18034-18066-18098-18130-18162-18194-18226-18258-18290-18322-18354-18386-18418-18450-18480-18512-20164-20196-20228-20260-20292-20324-20356-20388-20420-20452-20484-20516-20548-20580-20612-20644-20676-20708-20740-20770-20802-22454-22486-22518-22550-22582-22614-22646-22678-22710-22742-22774-22806-22838-22870-22902-22934-22966-22998-23030-23060-23092},23092,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 80,9223372036854775807,0x12c580204800,0,DefaultEnhancedAscendMemoryPool,512,6291456,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,78,PrepareDataForSequenceAndScalarValue,Default/data-635,,{241-256-329-642-2526-2541-2614-2927-4816-4831-4904-5217-7106-7121-7194-7507-9396-9411-9484-9797-11686-11701-11774-12087-13976-13991-14064-14377-16266-16281-16354-16667-18556-18571-18644-18957-20846-20861-20934-21247},21247,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 82,9223372036854775807,0x12c580204a00,0,DefaultEnhancedAscendMemoryPool,512,6291456,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,80,PrepareDataForSequenceAndScalarValue,Default/data-636,,{241-2526-4816-7106-9396-11686-13976-16266-18556-20846},20846,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 84,9223372036854775807,0x12c580204c00,0,DefaultEnhancedAscendMemoryPool,512,6291456,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,82,SyncTensorData,Default/data-637,,{},,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 86,9223372036854775807,0x12c580204e00,0,DefaultEnhancedAscendMemoryPool,512,6291456,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,84,SyncTensorData,Default/data-638,,{1510-3795-6085-8375-10665-12955-15245-17535-19825-22115},22115,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 88,9223372036854775807,0x12c580205000,0,DefaultEnhancedAscendMemoryPool,512,6291456,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,86,PrepareDataForSequenceAndScalarValue,Default/data-639,,{241-241-241-241-241-243-243-256-256-256-256-256-280-329-329-329-329-329-373-373-373-520-520-520-642-642-642-642-642-838-838-838-1079-1079-1079-1299-1307-1315-1323-1331-1339-1347-1355-1363-1371-1379-1387-1395-1403-1411-1419-1427-1435-1443-1451-1461-2526-2526-2526-2526-2526-2528-2528-2541-2541-2541-2541-2541-2565-2614-2614-2614-2614-2614-2658-2658-2658-2805-2805-2805-2927-2927-2927-2927-2927-3123-3123-3123-3364-3364-3364-3584-3592-3600-3608-3616-3624-3632-3640-3648-3656-3664-3672-3680-3688-3696-3704-3712-3720-3728-3736-3746-4816-4816-4816-4816-4816-4818-4818-4831-4831-4831-4831-4831-4855-4904-4904-4904-4904-4904-4948-4948-4948-5095-5095-5095-5217-5217-5217-5217-5217-5413-5413-5413-5654-5654-5654-5874-5882-5890-5898-5906-5914-5922-5930-5938-5946-5954-5962-5970-5978-5986-5994-6002-6010-6018-6026-6036-7106-7106-7106-7106-7106-7108-7108-7121-7121-7121-7121-7121-7145-7194-7194-7194-7194-7194-7238-7238-7238-7385-7385-7385-7507-7507-7507-7507-7507-7703-7703-7703-7944-7944-7944-8164-8172-8180-8188-8196-8204-8212-8220-8228-8236-8244-8252-8260-8268-8276-8284-8292-8300-8308-8316-8326-9396-9396-9396-9396-9396-9398-9398-9411-9411-9411-9411-9411-9435-9484-9484-9484-9484-9484-9528-9528-9528-9675-9675-9675-9797-9797-9797-9797-9797-9993-9993-9993-10234-10234-10234-10454-10462-10470-10478-10486-10494-10502-10510-10518-10526-10534-10542-10550-10558-10566-10574-10582-10590-10598-10606-10616-11686-11686-11686-11686-11686-11688-11688-11701-11701-11701-11701-11701-11725-11774-11774-11774-11774-11774-11818-11818-11818-11965-11965-11965-12087-12087-12087-12087-12087-12283-12283-12283-12524-12524-12524-12744-12752-12760-12768-12776-12784-12792-12800-12808-12816-12824-12832-12840-12848-12856-12864-12872-12880-12888-12896-12906-13976-13976-13976-13976-13976-13978-13978-13991-13991-13991-13991-13991-14015-14064-14064-14064-14064-14064-14108-14108-14108-14255-14255-14255-14377-14377-14377-14377-14377-14573-14573-14573-14814-14814-14814-15034-15042-15050-15058-15066-15074-15082-15090-15098-15106-15114-15122-15130-15138-15146-15154-15162-15170-15178-15186-15196-16266-16266-16266-16266-16266-16268-16268-16281-16281-16281-16281-16281-16305-16354-16354-16354-16354-16354-16398-16398-16398-16545-16545-16545-16667-16667-16667-16667-16667-16863-16863-16863-17104-17104-17104-17324-17332-17340-17348-17356-17364-17372-17380-17388-17396-17404-17412-17420-17428-17436-17444-17452-17460-17468-17476-17486-18556-18556-18556-18556-18556-18558-18558-18571-18571-18571-18571-18571-18595-18644-18644-18644-18644-18644-18688-18688-18688-18835-18835-18835-18957-18957-18957-18957-18957-19153-19153-19153-19394-19394-19394-19614-19622-19630-19638-19646-19654-19662-19670-19678-19686-19694-19702-19710-19718-19726-19734-19742-19750-19758-19766-19776-20846-20846-20846-20846-20846-20848-20848-20861-20861-20861-20861-20861-20885-20934-20934-20934-20934-20934-20978-20978-20978-21125-21125-21125-21247-21247-21247-21247-21247-21443-21443-21443-21684-21684-21684-21904-21912-21920-21928-21936-21944-21952-21960-21968-21976-21984-21992-22000-22008-22016-22024-22032-22040-22048-22056-22066},22066,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 90,9223372036854775807,0x12c580205200,0,DefaultEnhancedAscendMemoryPool,512,6291456,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,88,SyncTensorData,Default/data-640,,{1837-4122-6412-8702-10992-13282-15572-17862-20152-22442},22442,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 92,9223372036854775807,0x12c580205400,0,DefaultEnhancedAscendMemoryPool,512,6291456,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,90,SyncTensorData,Default/data-641,,{1612-3897-6187-8477-10767-13057-15347-17637-19927-22217},22217,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 94,9223372036854775807,0x12c580205600,0,DefaultEnhancedAscendMemoryPool,512,6291456,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,92,PrepareDataForSequenceAndScalarValue,Default/data-642,,{646-1220-2931-3505-5221-5795-7511-8085-9801-10375-12091-12665-14381-14955-16671-17245-18961-19535-21251-21825},21825,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 96,9223372036854775807,0x12c580205800,0,DefaultEnhancedAscendMemoryPool,524800,6291456,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,94,SyncTensorData,Default/data-643,,{256-2541-4831-7121-9411-11701-13991-16281-18571-20861},20861,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 98,9223372036854775807,0x12c580285a00,0,DefaultEnhancedAscendMemoryPool,512,6291456,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,96,SyncTensorData,Default/data-644,,{1630-3915-6205-8495-10785-13075-15365-17655-19945-22235},22235,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 100,9223372036854775807,0x12c580285c00,0,DefaultEnhancedAscendMemoryPool,512,6291456,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,98,PrepareDataForSequenceAndScalarValue,Default/data-645,,{373-520-838-1079-2658-2805-3123-3364-4948-5095-5413-5654-7238-7385-7703-7944-9528-9675-9993-10234-11818-11965-12283-12524-14108-14255-14573-14814-16398-16545-16863-17104-18688-18835-19153-19394-20978-21125-21443-21684},21684,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 102,9223372036854775807,0x12c580285e00,0,DefaultEnhancedAscendMemoryPool,512,6291456,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,100,PrepareDataForSequenceAndScalarValue,Default/data-646,,{373-520-838-1079-2658-2805-3123-3364-4948-5095-5413-5654-7238-7385-7703-7944-9528-9675-9993-10234-11818-11965-12283-12524-14108-14255-14573-14814-16398-16545-16863-17104-18688-18835-19153-19394-20978-21125-21443-21684},21684,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 104,9223372036854775807,0x12c580286000,0,DefaultEnhancedAscendMemoryPool,512,6291456,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,102,SyncTensorData,Default/data-647,,{1819-4104-6394-8684-10974-13264-15554-17844-20134-22424},22424,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 106,9223372036854775807,0x12c580286200,0,DefaultEnhancedAscendMemoryPool,512,6291456,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,104,PrepareDataForSequenceAndScalarValue,Default/data-648,,{744-749-990-995-3029-3034-3275-3280-5319-5324-5565-5570-7609-7614-7855-7860-9899-9904-10145-10150-12189-12194-12435-12440-14479-14484-14725-14730-16769-16774-17015-17020-19059-19064-19305-19310-21349-21354-21595-21600},21600,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 108,9223372036854775807,0x12c580286400,0,DefaultEnhancedAscendMemoryPool,512,6291456,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,106,PrepareDataForSequenceAndScalarValue,Default/data-649,,{292-379-401-442-526-548-589-2577-2664-2686-2727-2811-2833-2874-4867-4954-4976-5017-5101-5123-5164-7157-7244-7266-7307-7391-7413-7454-9447-9534-9556-9597-9681-9703-9744-11737-11824-11846-11887-11971-11993-12034-14027-14114-14136-14177-14261-14283-14324-16317-16404-16426-16467-16551-16573-16614-18607-18694-18716-18757-18841-18863-18904-20897-20984-21006-21047-21131-21153-21194},21194,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 110,9223372036854775807,0x12c580286600,0,DefaultEnhancedAscendMemoryPool,512,6291456,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,108,PrepareDataForSequenceAndScalarValue,Default/data-650,,{299-314-337-449-464-509-827-1068-2584-2599-2622-2734-2749-2794-3112-3353-4874-4889-4912-5024-5039-5084-5402-5643-7164-7179-7202-7314-7329-7374-7692-7933-9454-9469-9492-9604-9619-9664-9982-10223-11744-11759-11782-11894-11909-11954-12272-12513-14034-14049-14072-14184-14199-14244-14562-14803-16324-16339-16362-16474-16489-16534-16852-17093-18614-18629-18652-18764-18779-18824-19142-19383-20904-20919-20942-21054-21069-21114-21432-21673},21673,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 112,9223372036854775807,0x12c580286800,0,DefaultEnhancedAscendMemoryPool,512,6291456,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,110,SyncTensorData,Default/data-651,,{1606-1624-3891-3909-6181-6199-8471-8489-10761-10779-13051-13069-15341-15359-17631-17649-19921-19939-22211-22229},22229,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 114,9223372036854775807,0x12c580286a00,0,DefaultEnhancedAscendMemoryPool,512,6291456,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,112,SyncTensorData,Default/data-652,,{245-2530-4820-7110-9400-11690-13980-16270-18560-20850},20850,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 116,9223372036854775807,0x12c580286c00,0,DefaultEnhancedAscendMemoryPool,512,6291456,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,114,SyncTensorData,Default/data-653,,{1597-3882-6172-8462-10752-13042-15332-17622-19912-22202},22202,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 118,9223372036854775807,0x12c580286e00,0,DefaultEnhancedAscendMemoryPool,512,6291456,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,116,PrepareDataForSequenceAndScalarValue,Default/data-654,,{257-330-2542-2615-4832-4905-7122-7195-9412-9485-11702-11775-13992-14065-16282-16355-18572-18645-20862-20935},20935,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 120,9223372036854775807,0x12c580287000,0,DefaultEnhancedAscendMemoryPool,512,6291456,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,118,PrepareDataForSequenceAndScalarValue,Default/data-655,,{373-520-838-1079-2658-2805-3123-3364-4948-5095-5413-5654-7238-7385-7703-7944-9528-9675-9993-10234-11818-11965-12283-12524-14108-14255-14573-14814-16398-16545-16863-17104-18688-18835-19153-19394-20978-21125-21443-21684},21684,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 122,9223372036854775807,0x12c580287200,0,DefaultEnhancedAscendMemoryPool,512,6291456,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,120,PrepareDataForSequenceAndScalarValue,Default/data-656,,{297-312-335-384-406-413-425-447-462-507-531-553-560-572-594-601-612-691-729-756-770-816-867-888-905-919-933-975-1002-1016-1057-1109-1132-1149-1163-1177-2582-2597-2620-2669-2691-2698-2710-2732-2747-2792-2816-2838-2845-2857-2879-2886-2897-2976-3014-3041-3055-3101-3152-3173-3190-3204-3218-3260-3287-3301-3342-3394-3417-3434-3448-3462-4872-4887-4910-4959-4981-4988-5000-5022-5037-5082-5106-5128-5135-5147-5169-5176-5187-5266-5304-5331-5345-5391-5442-5463-5480-5494-5508-5550-5577-5591-5632-5684-5707-5724-5738-5752-7162-7177-7200-7249-7271-7278-7290-7312-7327-7372-7396-7418-7425-7437-7459-7466-7477-7556-7594-7621-7635-7681-7732-7753-7770-7784-7798-7840-7867-7881-7922-7974-7997-8014-8028-8042-9452-9467-9490-9539-9561-9568-9580-9602-9617-9662-9686-9708-9715-9727-9749-9756-9767-9846-9884-9911-9925-9971-10022-10043-10060-10074-10088-10130-10157-10171-10212-10264-10287-10304-10318-10332-11742-11757-11780-11829-11851-11858-11870-11892-11907-11952-11976-11998-12005-12017-12039-12046-12057-12136-12174-12201-12215-12261-12312-12333-12350-12364-12378-12420-12447-12461-12502-12554-12577-12594-12608-12622-14032-14047-14070-14119-14141-14148-14160-14182-14197-14242-14266-14288-14295-14307-14329-14336-14347-14426-14464-14491-14505-14551-14602-14623-14640-14654-14668-14710-14737-14751-14792-14844-14867-14884-14898-14912-16322-16337-16360-16409-16431-16438-16450-16472-16487-16532-16556-16578-16585-16597-16619-16626-16637-16716-16754-16781-16795-16841-16892-16913-16930-16944-16958-17000-17027-17041-17082-17134-17157-17174-17188-17202-18612-18627-18650-18699-18721-18728-18740-18762-18777-18822-18846-18868-18875-18887-18909-18916-18927-19006-19044-19071-19085-19131-19182-19203-19220-19234-19248-19290-19317-19331-19372-19424-19447-19464-19478-19492-20902-20917-20940-20989-21011-21018-21030-21052-21067-21112-21136-21158-21165-21177-21199-21206-21217-21296-21334-21361-21375-21421-21472-21493-21510-21524-21538-21580-21607-21621-21662-21714-21737-21754-21768-21782},21782,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 124,9223372036854775807,0x12c580287400,0,DefaultEnhancedAscendMemoryPool,512,6291456,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,122,SyncTensorData,Default/data-657,,{1637-1645-1653-1661-1669-1677-1685-1693-1701-1709-1717-1725-1733-1741-1749-1757-1765-1773-1781-1791-1799-3922-3930-3938-3946-3954-3962-3970-3978-3986-3994-4002-4010-4018-4026-4034-4042-4050-4058-4066-4076-4084-6212-6220-6228-6236-6244-6252-6260-6268-6276-6284-6292-6300-6308-6316-6324-6332-6340-6348-6356-6366-6374-8502-8510-8518-8526-8534-8542-8550-8558-8566-8574-8582-8590-8598-8606-8614-8622-8630-8638-8646-8656-8664-10792-10800-10808-10816-10824-10832-10840-10848-10856-10864-10872-10880-10888-10896-10904-10912-10920-10928-10936-10946-10954-13082-13090-13098-13106-13114-13122-13130-13138-13146-13154-13162-13170-13178-13186-13194-13202-13210-13218-13226-13236-13244-15372-15380-15388-15396-15404-15412-15420-15428-15436-15444-15452-15460-15468-15476-15484-15492-15500-15508-15516-15526-15534-17662-17670-17678-17686-17694-17702-17710-17718-17726-17734-17742-17750-17758-17766-17774-17782-17790-17798-17806-17816-17824-19952-19960-19968-19976-19984-19992-20000-20008-20016-20024-20032-20040-20048-20056-20064-20072-20080-20088-20096-20106-20114-22242-22250-22258-22266-22274-22282-22290-22298-22306-22314-22322-22330-22338-22346-22354-22362-22370-22378-22386-22396-22404},22404,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 126,9223372036854775807,0x12c580287600,0,DefaultEnhancedAscendMemoryPool,512,6291456,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,124,SyncTensorData,Default/data-658,,{646-2931-5221-7511-9801-12091-14381-16671-18961-21251},21251,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 128,9223372036854775807,0x12c580287800,0,DefaultEnhancedAscendMemoryPool,512,6291456,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,126,PrepareDataForSequenceAndScalarValue,Default/data-659,,{247-253-267-294-309-332-356-366-381-399-403-410-422-440-444-459-486-501-504-528-546-550-557-569-587-591-658-719-802-874-895-966-1048-1116-1139-1210-2532-2538-2552-2579-2594-2617-2641-2651-2666-2684-2688-2695-2707-2725-2729-2744-2771-2786-2789-2813-2831-2835-2842-2854-2872-2876-2943-3004-3087-3159-3180-3251-3333-3401-3424-3495-4822-4828-4842-4869-4884-4907-4931-4941-4956-4974-4978-4985-4997-5015-5019-5034-5061-5076-5079-5103-5121-5125-5132-5144-5162-5166-5233-5294-5377-5449-5470-5541-5623-5691-5714-5785-7112-7118-7132-7159-7174-7197-7221-7231-7246-7264-7268-7275-7287-7305-7309-7324-7351-7366-7369-7393-7411-7415-7422-7434-7452-7456-7523-7584-7667-7739-7760-7831-7913-7981-8004-8075-9402-9408-9422-9449-9464-9487-9511-9521-9536-9554-9558-9565-9577-9595-9599-9614-9641-9656-9659-9683-9701-9705-9712-9724-9742-9746-9813-9874-9957-10029-10050-10121-10203-10271-10294-10365-11692-11698-11712-11739-11754-11777-11801-11811-11826-11844-11848-11855-11867-11885-11889-11904-11931-11946-11949-11973-11991-11995-12002-12014-12032-12036-12103-12164-12247-12319-12340-12411-12493-12561-12584-12655-13982-13988-14002-14029-14044-14067-14091-14101-14116-14134-14138-14145-14157-14175-14179-14194-14221-14236-14239-14263-14281-14285-14292-14304-14322-14326-14393-14454-14537-14609-14630-14701-14783-14851-14874-14945-16272-16278-16292-16319-16334-16357-16381-16391-16406-16424-16428-16435-16447-16465-16469-16484-16511-16526-16529-16553-16571-16575-16582-16594-16612-16616-16683-16744-16827-16899-16920-16991-17073-17141-17164-17235-18562-18568-18582-18609-18624-18647-18671-18681-18696-18714-18718-18725-18737-18755-18759-18774-18801-18816-18819-18843-18861-18865-18872-18884-18902-18906-18973-19034-19117-19189-19210-19281-19363-19431-19454-19525-20852-20858-20872-20899-20914-20937-20961-20971-20986-21004-21008-21015-21027-21045-21049-21064-21091-21106-21109-21133-21151-21155-21162-21174-21192-21196-21263-21324-21407-21479-21500-21571-21653-21721-21744-21815},21815,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 130,9223372036854775807,0x12c580287a00,0,DefaultEnhancedAscendMemoryPool,524800,6291456,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,128,SyncTensorData,Default/data-660,,{329-2614-4904-7194-9484-11774-14064-16354-18644-20934},20934,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 132,9223372036854775807,0x12c580307c00,0,DefaultEnhancedAscendMemoryPool,512,6291456,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,130,PrepareDataForSequenceAndScalarValue,Default/data-661,,{290-2575-4865-7155-9445-11735-14025-16315-18605-20895},20895,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 134,9223372036854775807,0x12c580307e00,0,DefaultEnhancedAscendMemoryPool,512,6291456,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,132,SyncTensorData,Default/data-598,,{1845-4130-6420-8710-11000-13290-15580-17870-20160-22450},22450,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 136,9223372036854775807,0x12c580308000,0,DefaultEnhancedAscendMemoryPool,512,6291456,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,134,PrepareDataForSequenceAndScalarValue,Default/data-662,,{642-2927-5217-7507-9797-12087-14377-16667-18957-21247},21247,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 138,9223372036854775807,0x12c580308200,0,DefaultEnhancedAscendMemoryPool,512,6291456,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,136,SyncTensorData,Default/data-663,,{812-3097-5387-7677-9967-12257-14547-16837-19127-21417},21417,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 140,9223372036854775807,0x12c580308400,0,DefaultEnhancedAscendMemoryPool,512,6291456,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,138,SyncTensorData,Default/data-664,,{1810-4095-6385-8675-10965-13255-15545-17835-20125-22415},22415,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 142,9223372036854775807,0x12c580308600,0,DefaultEnhancedAscendMemoryPool,512,6291456,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,140,SyncTensorData,Default/data-665,,{276-2561-4851-7141-9431-11721-14011-16301-18591-20881},20881,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 144,9223372036854775807,0x12c580308800,0,DefaultEnhancedAscendMemoryPool,512,6291456,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,142,PrepareDataForSequenceAndScalarValue,Default/data-666,,{408-417-555-564-740-986-2693-2702-2840-2849-3025-3271-4983-4992-5130-5139-5315-5561-7273-7282-7420-7429-7605-7851-9563-9572-9710-9719-9895-10141-11853-11862-12000-12009-12185-12431-14143-14152-14290-14299-14475-14721-16433-16442-16580-16589-16765-17011-18723-18732-18870-18879-19055-19301-21013-21022-21160-21169-21345-21591},21591,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 146,9223372036854775807,0x12c580308a00,0,DefaultEnhancedAscendMemoryPool,512,6291456,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,144,SyncTensorData,Default/data-667,,{},,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 148,9223372036854775807,0x12c580308c00,0,DefaultEnhancedAscendMemoryPool,512,6291456,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,146,SyncTensorData,Default/data-668,,{722-3007-5297-7587-9877-12167-14457-16747-19037-21327},21327,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 150,9223372036854775807,0x12c580308e00,0,DefaultEnhancedAscendMemoryPool,512,6291456,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,148,PrepareDataForSequenceAndScalarValue,Default/data-669,,{250-259-306-326-391-432-456-471-538-579-597-623-695-705-708-733-759-774-788-791-820-850-853-908-922-937-952-955-979-1005-1020-1034-1037-1061-1092-1095-1152-1166-1181-1196-1199-1217-1595-2535-2544-2591-2611-2676-2717-2741-2756-2823-2864-2882-2908-2980-2990-2993-3018-3044-3059-3073-3076-3105-3135-3138-3193-3207-3222-3237-3240-3264-3290-3305-3319-3322-3346-3377-3380-3437-3451-3466-3481-3484-3502-3880-4825-4834-4881-4901-4966-5007-5031-5046-5113-5154-5172-5198-5270-5280-5283-5308-5334-5349-5363-5366-5395-5425-5428-5483-5497-5512-5527-5530-5554-5580-5595-5609-5612-5636-5667-5670-5727-5741-5756-5771-5774-5792-6170-7115-7124-7171-7191-7256-7297-7321-7336-7403-7444-7462-7488-7560-7570-7573-7598-7624-7639-7653-7656-7685-7715-7718-7773-7787-7802-7817-7820-7844-7870-7885-7899-7902-7926-7957-7960-8017-8031-8046-8061-8064-8082-8460-9405-9414-9461-9481-9546-9587-9611-9626-9693-9734-9752-9778-9850-9860-9863-9888-9914-9929-9943-9946-9975-10005-10008-10063-10077-10092-10107-10110-10134-10160-10175-10189-10192-10216-10247-10250-10307-10321-10336-10351-10354-10372-10750-11695-11704-11751-11771-11836-11877-11901-11916-11983-12024-12042-12068-12140-12150-12153-12178-12204-12219-12233-12236-12265-12295-12298-12353-12367-12382-12397-12400-12424-12450-12465-12479-12482-12506-12537-12540-12597-12611-12626-12641-12644-12662-13040-13985-13994-14041-14061-14126-14167-14191-14206-14273-14314-14332-14358-14430-14440-14443-14468-14494-14509-14523-14526-14555-14585-14588-14643-14657-14672-14687-14690-14714-14740-14755-14769-14772-14796-14827-14830-14887-14901-14916-14931-14934-14952-15330-16275-16284-16331-16351-16416-16457-16481-16496-16563-16604-16622-16648-16720-16730-16733-16758-16784-16799-16813-16816-16845-16875-16878-16933-16947-16962-16977-16980-17004-17030-17045-17059-17062-17086-17117-17120-17177-17191-17206-17221-17224-17242-17620-18565-18574-18621-18641-18706-18747-18771-18786-18853-18894-18912-18938-19010-19020-19023-19048-19074-19089-19103-19106-19135-19165-19168-19223-19237-19252-19267-19270-19294-19320-19335-19349-19352-19376-19407-19410-19467-19481-19496-19511-19514-19532-19910-20855-20864-20911-20931-20996-21037-21061-21076-21143-21184-21202-21228-21300-21310-21313-21338-21364-21379-21393-21396-21425-21455-21458-21513-21527-21542-21557-21560-21584-21610-21625-21639-21642-21666-21697-21700-21757-21771-21786-21801-21804-21822-22200},22200,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 152,9223372036854775807,0x12c580309000,0,DefaultEnhancedAscendMemoryPool,512,6291456,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,150,SyncTensorData,Default/data-670,,{1600-3885-6175-8465-10755-13045-15335-17625-19915-22205},22205,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 154,9223372036854775807,0x12c580309200,0,DefaultEnhancedAscendMemoryPool,512,6291456,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,152,SyncTensorData,Default/data-671,,{270-2555-4845-7135-9425-11715-14005-16295-18585-20875},20875,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 156,9223372036854775807,0x12c580309400,0,DefaultEnhancedAscendMemoryPool,66048,6291456,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,154,SyncTensorData,Default/data-672,,{323-346-476-491-867-888-1109-1132-2608-2631-2761-2776-3152-3173-3394-3417-4898-4921-5051-5066-5442-5463-5684-5707-7188-7211-7341-7356-7732-7753-7974-7997-9478-9501-9631-9646-10022-10043-10264-10287-11768-11791-11921-11936-12312-12333-12554-12577-14058-14081-14211-14226-14602-14623-14844-14867-16348-16371-16501-16516-16892-16913-17134-17157-18638-18661-18791-18806-19182-19203-19424-19447-20928-20951-21081-21096-21472-21493-21714-21737},21737,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 158,9223372036854775807,0x12c580319600,0,DefaultEnhancedAscendMemoryPool,512,6291456,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,156,SyncTensorData,Default/data-673,,{646-725-1507-2931-3010-3792-5221-5300-6082-7511-7590-8372-9801-9880-10662-12091-12170-12952-14381-14460-15242-16671-16750-17532-18961-19040-19822-21251-21330-22112},22112,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 160,9223372036854775807,0x12c580319800,0,DefaultEnhancedAscendMemoryPool,512,6291456,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,158,SyncTensorData,Default/data-674,,{1847-1879-1911-1943-1975-2007-2039-2071-2103-2135-2167-2199-2231-2263-2295-2327-2359-2391-2423-2457-2489-4132-4164-4196-4228-4260-4292-4324-4356-4388-4420-4452-4484-4516-4548-4580-4612-4644-4676-4708-4742-4774-6422-6454-6486-6518-6550-6582-6614-6646-6678-6710-6742-6774-6806-6838-6870-6902-6934-6966-6998-7032-7064-8712-8744-8776-8808-8840-8872-8904-8936-8968-9000-9032-9064-9096-9128-9160-9192-9224-9256-9288-9322-9354-11002-11034-11066-11098-11130-11162-11194-11226-11258-11290-11322-11354-11386-11418-11450-11482-11514-11546-11578-11612-11644-13292-13324-13356-13388-13420-13452-13484-13516-13548-13580-13612-13644-13676-13708-13740-13772-13804-13836-13868-13902-13934-15582-15614-15646-15678-15710-15742-15774-15806-15838-15870-15902-15934-15966-15998-16030-16062-16094-16126-16158-16192-16224-17872-17904-17936-17968-18000-18032-18064-18096-18128-18160-18192-18224-18256-18288-18320-18352-18384-18416-18448-18482-18514-20162-20194-20226-20258-20290-20322-20354-20386-20418-20450-20482-20514-20546-20578-20610-20642-20674-20706-20738-20772-20804-22452-22484-22516-22548-22580-22612-22644-22676-22708-22740-22772-22804-22836-22868-22900-22932-22964-22996-23028-23062-23094},23094,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 162,9223372036854775807,0x12c580319a00,0,DefaultEnhancedAscendMemoryPool,512,6291456,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,160,SyncTensorData,Default/data-675,,{608-1224-1496-1498-1504-2893-3509-3781-3783-3789-5183-5799-6071-6073-6079-7473-8089-8361-8363-8369-9763-10379-10651-10653-10659-12053-12669-12941-12943-12949-14343-14959-15231-15233-15239-16633-17249-17521-17523-17529-18923-19539-19811-19813-19819-21213-21829-22101-22103-22109},22109,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 164,9223372036854775807,0x12c580319c00,0,DefaultEnhancedAscendMemoryPool,512,6291456,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,162,PrepareDataForSequenceAndScalarValue,Default/data-676,,{633-646-2918-2931-5208-5221-7498-7511-9788-9801-12078-12091-14368-14381-16658-16671-18948-18961-21238-21251},21251,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 166,9223372036854775807,0x12c580319e00,0,DefaultEnhancedAscendMemoryPool,512,6291456,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,164,SyncTensorData,Default/data-677,,{1523-3808-6098-8388-10678-12968-15258-17548-19838-22128},22128,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 168,9223372036854775807,0x12c58031a000,0,DefaultEnhancedAscendMemoryPool,512,6291456,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,166,PrepareDataForSequenceAndScalarValue,Default/data-678,,{1789-4074-6364-8654-10944-13234-15524-17814-20104-22394},22394,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 170,9223372036854775807,0x12c58031a200,0,DefaultEnhancedAscendMemoryPool,512,6291456,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,168,PrepareDataForSequenceAndScalarValue,Default/data-679,,{241-256-329-2526-2541-2614-4816-4831-4904-7106-7121-7194-9396-9411-9484-11686-11701-11774-13976-13991-14064-16266-16281-16354-18556-18571-18644-20846-20861-20934},20934,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 172,9223372036854775807,0x12c58031a400,0,DefaultEnhancedAscendMemoryPool,512,6291456,mindspore/ccsrc/runtime/core/actors/base/data_prepare_actor.cc,71,ConstantValue,170,SyncTensorData,Default/data-680,,{1807-4092-6382-8672-10962-13252-15542-17832-20122-22412},22412,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 173,9223372036854775807,0x12c58031a600,0,DefaultEnhancedAscendMemoryPool,512,6291456,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,172,kernel_graph_0_DataPrepareActor,scale_sense,kernel_graph_0_DataPrepareActor,{608-722-1224-1489-1592-2893-3007-3509-3774-3877-5183-5297-5799-6064-6167-7473-7587-8089-8354-8457-9763-9877-10379-10644-10747-12053-12167-12669-12934-13037-14343-14457-14959-15224-15327-16633-16747-17249-17514-17617-18923-19037-19539-19804-19907-21213-21327-21829-22094-22197},22197,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 174,9223372036854775807,0x12c58031a800,0,DefaultEnhancedAscendMemoryPool,16896,6291456,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,173,kernel_graph_0_DataPrepareActor,model.norm_out.weight,kernel_graph_0_DataPrepareActor,{583-713-2511-2513-2868-2998-4796-4798-5158-5288-7086-7088-7448-7578-9376-9378-9738-9868-11666-11668-12028-12158-13956-13958-14318-14448-16246-16248-16608-16738-18536-18538-18898-19028-20826-20828-21188-21318-23116-23118},23118,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 175,9223372036854775807,0x12c58031ea00,0,DefaultEnhancedAscendMemoryPool,180355584,186646528,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,174,kernel_graph_0_DataPrepareActor,model.layers.1.feed_forward.w2.weight,kernel_graph_0_DataPrepareActor,{569-2286-2292-2854-4571-4577-5144-6861-6867-7434-9151-9157-9724-11441-11447-12014-13731-13737-14304-16021-16027-16594-18311-18317-18884-20601-20607-21174-22891-22897},22897,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 176,9223372036854775807,0x12c58af1ec00,0,DefaultEnhancedAscendMemoryPool,180355584,367001600,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,175,kernel_graph_0_DataPrepareActor,model.layers.1.feed_forward.w1.weight,kernel_graph_0_DataPrepareActor,{557-2254-2260-2842-4539-4545-5132-6829-6835-7422-9119-9125-9712-11409-11415-12002-13699-13705-14292-15989-15995-16582-18279-18285-18872-20569-20575-21162-22859-22865},22865,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 177,9223372036854775807,0x12c595b1ee00,0,DefaultEnhancedAscendMemoryPool,16896,367001600,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,176,kernel_graph_0_DataPrepareActor,model.layers.1.ffn_norm.weight,kernel_graph_0_DataPrepareActor,{542-796-2446-2452-2827-3081-4731-4737-5117-5371-7021-7027-7407-7661-9311-9317-9697-9951-11601-11607-11987-12241-13891-13897-14277-14531-16181-16187-16567-16821-18471-18477-18857-19111-20761-20767-21147-21401-23051-23057},23057,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 178,9223372036854775807,0x12c595b23000,0,DefaultEnhancedAscendMemoryPool,67109376,434110464,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,177,kernel_graph_0_DataPrepareActor,model.layers.1.attention.wo.weight,kernel_graph_0_DataPrepareActor,{528-2222-2228-2813-4507-4513-5103-6797-6803-7393-9087-9093-9683-11377-11383-11973-13667-13673-14263-15957-15963-16553-18247-18253-18843-20537-20543-21133-22827-22833},22833,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 179,9223372036854775807,0x12c599b23200,0,DefaultEnhancedAscendMemoryPool,67109376,501219328,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,178,kernel_graph_0_DataPrepareActor,model.layers.1.attention.wv.weight,kernel_graph_0_DataPrepareActor,{504-2190-2196-2789-4475-4481-5079-6765-6771-7369-9055-9061-9659-11345-11351-11949-13635-13641-14239-15925-15931-16529-18215-18221-18819-20505-20511-21109-22795-22801},22801,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 180,9223372036854775807,0x12c59db23400,0,DefaultEnhancedAscendMemoryPool,16896,501219328,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,179,kernel_graph_0_DataPrepareActor,model.layers.1.attention_norm.weight,kernel_graph_0_DataPrepareActor,{436-960-2478-2484-2721-3245-4763-4769-5011-5535-7053-7059-7301-7825-9343-9349-9591-10115-11633-11639-11881-12405-13923-13929-14171-14695-16213-16219-16461-16985-18503-18509-18751-19275-20793-20799-21041-21565-23083-23089},23089,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 181,9223372036854775807,0x12c59db27600,0,DefaultEnhancedAscendMemoryPool,180355584,681574400,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,180,kernel_graph_0_DataPrepareActor,model.layers.0.feed_forward.w2.weight,kernel_graph_0_DataPrepareActor,{422-2062-2068-2707-4347-4353-4997-6637-6643-7287-8927-8933-9577-11217-11223-11867-13507-13513-14157-15797-15803-16447-18087-18093-18737-20377-20383-21027-22667-22673},22673,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 182,9223372036854775807,0x12c5a8727800,0,DefaultEnhancedAscendMemoryPool,180355584,861929472,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,181,kernel_graph_0_DataPrepareActor,model.layers.0.feed_forward.w1.weight,kernel_graph_0_DataPrepareActor,{410-2030-2036-2695-4315-4321-4985-6605-6611-7275-8895-8901-9565-11185-11191-11855-13475-13481-14145-15765-15771-16435-18055-18061-18725-20345-20351-21015-22635-22641},22641,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 183,9223372036854775807,0x12c5b3327a00,0,DefaultEnhancedAscendMemoryPool,16896,861929472,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,182,kernel_graph_0_DataPrepareActor,model.layers.0.ffn_norm.weight,kernel_graph_0_DataPrepareActor,{395-1042-2382-2388-2680-3327-4667-4673-4970-5617-6957-6963-7260-7907-9247-9253-9550-10197-11537-11543-11840-12487-13827-13833-14130-14777-16117-16123-16420-17067-18407-18413-18710-19357-20697-20703-21000-21647-22987-22993},22993,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 184,9223372036854775807,0x12c5b332bc00,0,DefaultEnhancedAscendMemoryPool,67109376,929038336,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,183,kernel_graph_0_DataPrepareActor,model.layers.0.attention.wo.weight,kernel_graph_0_DataPrepareActor,{381-1998-2004-2666-4283-4289-4956-6573-6579-7246-8863-8869-9536-11153-11159-11826-13443-13449-14116-15733-15739-16406-18023-18029-18696-20313-20319-20986-22603-22609},22609,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 185,9223372036854775807,0x12c5b732be00,0,DefaultEnhancedAscendMemoryPool,67109376,996147200,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,184,kernel_graph_0_DataPrepareActor,model.layers.0.attention.wv.weight,kernel_graph_0_DataPrepareActor,{332-1966-1972-2617-4251-4257-4907-6541-6547-7197-8831-8837-9487-11121-11127-11777-13411-13417-14067-15701-15707-16357-17991-17997-18647-20281-20287-20937-22571-22577},22577,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 186,9223372036854775807,0x12c5bb32c000,0,DefaultEnhancedAscendMemoryPool,16896,996147200,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,185,kernel_graph_0_DataPrepareActor,model.layers.0.attention_norm.weight,kernel_graph_0_DataPrepareActor,{263-1204-2414-2420-2548-3489-4699-4705-4838-5779-6989-6995-7128-8069-9279-9285-9418-10359-11569-11575-11708-12649-13859-13865-13998-14939-16149-16155-16288-17229-18439-18445-18578-19519-20729-20735-20868-21809-23019-23025},23025,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 187,9223372036854775807,0x12c5bb330200,0,DefaultEnhancedAscendMemoryPool,524288512,1520435200,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,186,kernel_graph_0_DataPrepareActor,model.tok_embeddings.embedding_weight,kernel_graph_0_DataPrepareActor,{243-1870-1876-2528-4155-4161-4818-6445-6451-7108-8735-8741-9398-11025-11031-11688-13315-13321-13978-15605-15611-16268-17895-17901-18558-20185-20191-20848-22475-22481},22481,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 188,9223372036854775807,0x12c5da730400,0,DefaultEnhancedAscendMemoryPool,67109376,1587544064,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,187,kernel_graph_0_DataPrepareActor,model.layers.0.attention.wk.weight,kernel_graph_0_DataPrepareActor,{294-1934-1940-2579-4219-4225-4869-6509-6515-7159-8799-8805-9449-11089-11095-11739-13379-13385-14029-15669-15675-16319-17959-17965-18609-20249-20255-20899-22539-22545},22545,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 189,9223372036854775807,0x12c5de730600,0,DefaultEnhancedAscendMemoryPool,67109376,1654652928,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,188,kernel_graph_0_DataPrepareActor,model.layers.0.attention.wq.weight,kernel_graph_0_DataPrepareActor,{309-1902-1908-2594-4187-4193-4884-6477-6483-7174-8767-8773-9464-11057-11063-11754-13347-13353-14044-15637-15643-16334-17927-17933-18624-20217-20223-20914-22507-22513},22513,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 190,9223372036854775807,0x12c5e2730800,0,DefaultEnhancedAscendMemoryPool,180355584,1835008000,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,189,kernel_graph_0_DataPrepareActor,model.layers.0.feed_forward.w3.weight,kernel_graph_0_DataPrepareActor,{403-2094-2100-2688-4379-4385-4978-6669-6675-7268-8959-8965-9558-11249-11255-11848-13539-13545-14138-15829-15835-16428-18119-18125-18718-20409-20415-21008-22699-22705},22705,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 191,9223372036854775807,0x12c5ed330a00,0,DefaultEnhancedAscendMemoryPool,67109376,1902116864,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,190,kernel_graph_0_DataPrepareActor,model.layers.1.attention.wk.weight,kernel_graph_0_DataPrepareActor,{444-2158-2164-2729-4443-4449-5019-6733-6739-7309-9023-9029-9599-11313-11319-11889-13603-13609-14179-15893-15899-16469-18183-18189-18759-20473-20479-21049-22763-22769},22769,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 192,9223372036854775807,0x12c5f1330c00,0,DefaultEnhancedAscendMemoryPool,67109376,1969225728,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,191,kernel_graph_0_DataPrepareActor,model.layers.1.attention.wq.weight,kernel_graph_0_DataPrepareActor,{459-2126-2132-2744-4411-4417-5034-6701-6707-7324-8991-8997-9614-11281-11287-11904-13571-13577-14194-15861-15867-16484-18151-18157-18774-20441-20447-21064-22731-22737},22737,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 193,9223372036854775807,0x12c5f5330e00,0,DefaultEnhancedAscendMemoryPool,180355584,2149580800,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,192,kernel_graph_0_DataPrepareActor,model.layers.1.feed_forward.w3.weight,kernel_graph_0_DataPrepareActor,{550-2318-2324-2835-4603-4609-5125-6893-6899-7415-9183-9189-9705-11473-11479-11995-13763-13769-14285-16053-16059-16575-18343-18349-18865-20633-20639-21155-22923-22929},22929,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 194,9223372036854775807,0x12c5fff31000,0,DefaultEnhancedAscendMemoryPool,524288512,2673868800,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,193,kernel_graph_0_DataPrepareActor,lm_head.weight,kernel_graph_0_DataPrepareActor,{591-2350-2356-2876-4635-4641-5166-6925-6931-7456-9215-9221-9746-11505-11511-12036-13795-13801-14326-16085-16091-16616-18375-18381-18906-20665-20671-21196-22955-22961},22961,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 195,9223372036854775807,0x12c61f331200,0,DefaultEnhancedAscendMemoryPool,512,2673868800,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,194,kernel_graph_0_DataPrepareActor,current_iterator_step,kernel_graph_0_DataPrepareActor,{809-1520-1523-1590-3094-3805-3808-3875-5384-6095-6098-6165-7674-8385-8388-8455-9964-10675-10678-10745-12254-12965-12968-13035-14544-15255-15258-15325-16834-17545-17548-17615-19124-19835-19838-19905-21414-22125-22128-22195},22195,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 196,9223372036854775807,0x12c61f331400,0,DefaultEnhancedAscendMemoryPool,512,2673868800,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,195,kernel_graph_0_DataPrepareActor,last_overflow_iterator_step,kernel_graph_0_DataPrepareActor,{809-1520-1588-3094-3805-3873-5384-6095-6163-7674-8385-8453-9964-10675-10743-12254-12965-13033-14544-15255-15323-16834-17545-17613-19124-19835-19903-21414-22125-22193},22193,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 197,9223372036854775807,0x12c61f331600,0,DefaultEnhancedAscendMemoryPool,512,2673868800,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,196,kernel_graph_0_DataPrepareActor,global_step,kernel_graph_0_DataPrepareActor,{1595-1789-1845-3880-4074-4130-6170-6364-6420-8460-8654-8710-10750-10944-11000-13040-13234-13290-15330-15524-15580-17620-17814-17870-19910-20104-20160-22200-22394-22450},22450,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 198,9223372036854775807,0x12c61f331800,0,DefaultEnhancedAscendMemoryPool,16896,2673868800,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,197,kernel_graph_0_DataPrepareActor,adam_v.model.norm_out.weight,kernel_graph_0_DataPrepareActor,{2487-2517-4772-4802-7062-7092-9352-9382-11642-11672-13932-13962-16222-16252-18512-18542-20802-20832-23092-23122},23122,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 199,9223372036854775807,0x12c61f335a00,0,DefaultEnhancedAscendMemoryPool,16896,2673868800,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,198,kernel_graph_0_DataPrepareActor,adam_v.model.layers.1.attention_norm.weight,kernel_graph_0_DataPrepareActor,{2455-2480-4740-4765-7030-7055-9320-9345-11610-11635-13900-13925-16190-16215-18480-18505-20770-20795-23060-23085},23085,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 200,9223372036854775807,0x12c61f339c00,0,DefaultEnhancedAscendMemoryPool,16896,2673868800,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,199,kernel_graph_0_DataPrepareActor,adam_v.model.layers.1.ffn_norm.weight,kernel_graph_0_DataPrepareActor,{2425-2448-4710-4733-7000-7023-9290-9313-11580-11603-13870-13893-16160-16183-18450-18473-20740-20763-23030-23053},23053,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 201,9223372036854775807,0x12c61f33de00,0,DefaultEnhancedAscendMemoryPool,16896,2673868800,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,200,kernel_graph_0_DataPrepareActor,adam_v.model.layers.0.attention_norm.weight,kernel_graph_0_DataPrepareActor,{2393-2416-4678-4701-6968-6991-9258-9281-11548-11571-13838-13861-16128-16151-18418-18441-20708-20731-22998-23021},23021,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 202,9223372036854775807,0x12c61f342000,0,DefaultEnhancedAscendMemoryPool,16896,2673868800,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,201,kernel_graph_0_DataPrepareActor,adam_v.model.layers.0.ffn_norm.weight,kernel_graph_0_DataPrepareActor,{2361-2384-4646-4669-6936-6959-9226-9249-11516-11539-13806-13829-16096-16119-18386-18409-20676-20699-22966-22989},22989,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 203,9223372036854775807,0x12c61f346200,0,DefaultEnhancedAscendMemoryPool,524288512,3198156800,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,202,kernel_graph_0_DataPrepareActor,adam_v.lm_head.weight,kernel_graph_0_DataPrepareActor,{2329-2352-4614-4637-6904-6927-9194-9217-11484-11507-13774-13797-16064-16087-18354-18377-20644-20667-22934-22957},22957,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 204,9223372036854775807,0x12c63e746400,0,DefaultEnhancedAscendMemoryPool,180355584,3378511872,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,203,kernel_graph_0_DataPrepareActor,adam_v.model.layers.1.feed_forward.w3.weight,kernel_graph_0_DataPrepareActor,{2297-2320-4582-4605-6872-6895-9162-9185-11452-11475-13742-13765-16032-16055-18322-18345-20612-20635-22902-22925},22925,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 205,9223372036854775807,0x12c649346600,0,DefaultEnhancedAscendMemoryPool,180355584,3558866944,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,204,kernel_graph_0_DataPrepareActor,adam_v.model.layers.1.feed_forward.w2.weight,kernel_graph_0_DataPrepareActor,{2265-2288-4550-4573-6840-6863-9130-9153-11420-11443-13710-13733-16000-16023-18290-18313-20580-20603-22870-22893},22893,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 206,9223372036854775807,0x12c653f46800,0,DefaultEnhancedAscendMemoryPool,180355584,3739222016,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,205,kernel_graph_0_DataPrepareActor,adam_v.model.layers.1.feed_forward.w1.weight,kernel_graph_0_DataPrepareActor,{2233-2256-4518-4541-6808-6831-9098-9121-11388-11411-13678-13701-15968-15991-18258-18281-20548-20571-22838-22861},22861,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 207,9223372036854775807,0x12c65eb46a00,0,DefaultEnhancedAscendMemoryPool,67109376,3806330880,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,206,kernel_graph_0_DataPrepareActor,adam_v.model.layers.1.attention.wo.weight,kernel_graph_0_DataPrepareActor,{2201-2224-4486-4509-6776-6799-9066-9089-11356-11379-13646-13669-15936-15959-18226-18249-20516-20539-22806-22829},22829,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 208,9223372036854775807,0x12c662b46c00,0,DefaultEnhancedAscendMemoryPool,67109376,3873439744,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,207,kernel_graph_0_DataPrepareActor,adam_v.model.layers.1.attention.wv.weight,kernel_graph_0_DataPrepareActor,{2169-2192-4454-4477-6744-6767-9034-9057-11324-11347-13614-13637-15904-15927-18194-18217-20484-20507-22774-22797},22797,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 209,9223372036854775807,0x12c666b46e00,0,DefaultEnhancedAscendMemoryPool,67109376,3940548608,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,208,kernel_graph_0_DataPrepareActor,adam_v.model.layers.1.attention.wk.weight,kernel_graph_0_DataPrepareActor,{2137-2160-4422-4445-6712-6735-9002-9025-11292-11315-13582-13605-15872-15895-18162-18185-20452-20475-22742-22765},22765,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 210,9223372036854775807,0x12c66ab47000,0,DefaultEnhancedAscendMemoryPool,67109376,4007657472,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,209,kernel_graph_0_DataPrepareActor,adam_v.model.layers.1.attention.wq.weight,kernel_graph_0_DataPrepareActor,{2105-2128-4390-4413-6680-6703-8970-8993-11260-11283-13550-13573-15840-15863-18130-18153-20420-20443-22710-22733},22733,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 211,9223372036854775807,0x12c66eb47200,0,DefaultEnhancedAscendMemoryPool,180355584,4188012544,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,210,kernel_graph_0_DataPrepareActor,adam_v.model.layers.0.feed_forward.w3.weight,kernel_graph_0_DataPrepareActor,{2073-2096-4358-4381-6648-6671-8938-8961-11228-11251-13518-13541-15808-15831-18098-18121-20388-20411-22678-22701},22701,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 212,9223372036854775807,0x12c679747400,0,DefaultEnhancedAscendMemoryPool,180355584,4368367616,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,211,kernel_graph_0_DataPrepareActor,adam_v.model.layers.0.feed_forward.w2.weight,kernel_graph_0_DataPrepareActor,{2041-2064-4326-4349-6616-6639-8906-8929-11196-11219-13486-13509-15776-15799-18066-18089-20356-20379-22646-22669},22669,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 213,9223372036854775807,0x12c684347600,0,DefaultEnhancedAscendMemoryPool,180355584,4548722688,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,212,kernel_graph_0_DataPrepareActor,adam_v.model.layers.0.feed_forward.w1.weight,kernel_graph_0_DataPrepareActor,{2009-2032-4294-4317-6584-6607-8874-8897-11164-11187-13454-13477-15744-15767-18034-18057-20324-20347-22614-22637},22637,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 214,9223372036854775807,0x12c68ef47800,0,DefaultEnhancedAscendMemoryPool,67109376,4615831552,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,213,kernel_graph_0_DataPrepareActor,adam_v.model.layers.0.attention.wo.weight,kernel_graph_0_DataPrepareActor,{1977-2000-4262-4285-6552-6575-8842-8865-11132-11155-13422-13445-15712-15735-18002-18025-20292-20315-22582-22605},22605,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 215,9223372036854775807,0x12c692f47a00,0,DefaultEnhancedAscendMemoryPool,67109376,4682940416,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,214,kernel_graph_0_DataPrepareActor,adam_v.model.layers.0.attention.wv.weight,kernel_graph_0_DataPrepareActor,{1945-1968-4230-4253-6520-6543-8810-8833-11100-11123-13390-13413-15680-15703-17970-17993-20260-20283-22550-22573},22573,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 216,9223372036854775807,0x12c696f47c00,0,DefaultEnhancedAscendMemoryPool,67109376,4750049280,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,215,kernel_graph_0_DataPrepareActor,adam_v.model.layers.0.attention.wk.weight,kernel_graph_0_DataPrepareActor,{1913-1936-4198-4221-6488-6511-8778-8801-11068-11091-13358-13381-15648-15671-17938-17961-20228-20251-22518-22541},22541,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 217,9223372036854775807,0x12c69af47e00,0,DefaultEnhancedAscendMemoryPool,67109376,4817158144,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,216,kernel_graph_0_DataPrepareActor,adam_v.model.layers.0.attention.wq.weight,kernel_graph_0_DataPrepareActor,{1881-1904-4166-4189-6456-6479-8746-8769-11036-11059-13326-13349-15616-15639-17906-17929-20196-20219-22486-22509},22509,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 218,9223372036854775807,0x12c69ef48000,0,DefaultEnhancedAscendMemoryPool,524288512,5341446144,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,217,kernel_graph_0_DataPrepareActor,adam_v.model.tok_embeddings.embedding_weight,kernel_graph_0_DataPrepareActor,{1849-1872-4134-4157-6424-6447-8714-8737-11004-11027-13294-13317-15584-15607-17874-17897-20164-20187-22454-22477},22477,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 219,9223372036854775807,0x12c6be348200,0,DefaultEnhancedAscendMemoryPool,524288512,5865734144,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,218,kernel_graph_0_DataPrepareActor,adam_m.model.tok_embeddings.embedding_weight,kernel_graph_0_DataPrepareActor,{1847-1874-4132-4159-6422-6449-8712-8739-11002-11029-13292-13319-15582-15609-17872-17899-20162-20189-22452-22479},22479,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 220,9223372036854775807,0x12c6dd748400,0,DefaultEnhancedAscendMemoryPool,67109376,5932843008,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,219,kernel_graph_0_DataPrepareActor,adam_m.model.layers.0.attention.wq.weight,kernel_graph_0_DataPrepareActor,{1879-1906-4164-4191-6454-6481-8744-8771-11034-11061-13324-13351-15614-15641-17904-17931-20194-20221-22484-22511},22511,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 221,9223372036854775807,0x12c6e1748600,0,DefaultEnhancedAscendMemoryPool,67109376,5999951872,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,220,kernel_graph_0_DataPrepareActor,adam_m.model.layers.0.attention.wk.weight,kernel_graph_0_DataPrepareActor,{1911-1938-4196-4223-6486-6513-8776-8803-11066-11093-13356-13383-15646-15673-17936-17963-20226-20253-22516-22543},22543,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 222,9223372036854775807,0x12c6e5748800,0,DefaultEnhancedAscendMemoryPool,67109376,6067060736,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,221,kernel_graph_0_DataPrepareActor,adam_m.model.layers.0.attention.wv.weight,kernel_graph_0_DataPrepareActor,{1943-1970-4228-4255-6518-6545-8808-8835-11098-11125-13388-13415-15678-15705-17968-17995-20258-20285-22548-22575},22575,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 223,9223372036854775807,0x12c6e9748a00,0,DefaultEnhancedAscendMemoryPool,67109376,6134169600,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,222,kernel_graph_0_DataPrepareActor,adam_m.model.layers.0.attention.wo.weight,kernel_graph_0_DataPrepareActor,{1975-2002-4260-4287-6550-6577-8840-8867-11130-11157-13420-13447-15710-15737-18000-18027-20290-20317-22580-22607},22607,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 224,9223372036854775807,0x12c6ed748c00,0,DefaultEnhancedAscendMemoryPool,180355584,6314524672,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,223,kernel_graph_0_DataPrepareActor,adam_m.model.layers.0.feed_forward.w1.weight,kernel_graph_0_DataPrepareActor,{2007-2034-4292-4319-6582-6609-8872-8899-11162-11189-13452-13479-15742-15769-18032-18059-20322-20349-22612-22639},22639,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 225,9223372036854775807,0x12c6f8348e00,0,DefaultEnhancedAscendMemoryPool,180355584,6494879744,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,224,kernel_graph_0_DataPrepareActor,adam_m.model.layers.0.feed_forward.w2.weight,kernel_graph_0_DataPrepareActor,{2039-2066-4324-4351-6614-6641-8904-8931-11194-11221-13484-13511-15774-15801-18064-18091-20354-20381-22644-22671},22671,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 226,9223372036854775807,0x12c702f49000,0,DefaultEnhancedAscendMemoryPool,180355584,6675234816,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,225,kernel_graph_0_DataPrepareActor,adam_m.model.layers.0.feed_forward.w3.weight,kernel_graph_0_DataPrepareActor,{2071-2098-4356-4383-6646-6673-8936-8963-11226-11253-13516-13543-15806-15833-18096-18123-20386-20413-22676-22703},22703,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 227,9223372036854775807,0x12c70db49200,0,DefaultEnhancedAscendMemoryPool,67109376,6742343680,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,226,kernel_graph_0_DataPrepareActor,adam_m.model.layers.1.attention.wq.weight,kernel_graph_0_DataPrepareActor,{2103-2130-4388-4415-6678-6705-8968-8995-11258-11285-13548-13575-15838-15865-18128-18155-20418-20445-22708-22735},22735,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 228,9223372036854775807,0x12c711b49400,0,DefaultEnhancedAscendMemoryPool,67109376,6809452544,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,227,kernel_graph_0_DataPrepareActor,adam_m.model.layers.1.attention.wk.weight,kernel_graph_0_DataPrepareActor,{2135-2162-4420-4447-6710-6737-9000-9027-11290-11317-13580-13607-15870-15897-18160-18187-20450-20477-22740-22767},22767,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 229,9223372036854775807,0x12c715b49600,0,DefaultEnhancedAscendMemoryPool,67109376,6876561408,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,228,kernel_graph_0_DataPrepareActor,adam_m.model.layers.1.attention.wv.weight,kernel_graph_0_DataPrepareActor,{2167-2194-4452-4479-6742-6769-9032-9059-11322-11349-13612-13639-15902-15929-18192-18219-20482-20509-22772-22799},22799,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 230,9223372036854775807,0x12c719b49800,0,DefaultEnhancedAscendMemoryPool,67109376,6943670272,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,229,kernel_graph_0_DataPrepareActor,adam_m.model.layers.1.attention.wo.weight,kernel_graph_0_DataPrepareActor,{2199-2226-4484-4511-6774-6801-9064-9091-11354-11381-13644-13671-15934-15961-18224-18251-20514-20541-22804-22831},22831,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 231,9223372036854775807,0x12c71db49a00,0,DefaultEnhancedAscendMemoryPool,180355584,7124025344,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,230,kernel_graph_0_DataPrepareActor,adam_m.model.layers.1.feed_forward.w1.weight,kernel_graph_0_DataPrepareActor,{2231-2258-4516-4543-6806-6833-9096-9123-11386-11413-13676-13703-15966-15993-18256-18283-20546-20573-22836-22863},22863,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 232,9223372036854775807,0x12c728749c00,0,DefaultEnhancedAscendMemoryPool,180355584,7304380416,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,231,kernel_graph_0_DataPrepareActor,adam_m.model.layers.1.feed_forward.w2.weight,kernel_graph_0_DataPrepareActor,{2263-2290-4548-4575-6838-6865-9128-9155-11418-11445-13708-13735-15998-16025-18288-18315-20578-20605-22868-22895},22895,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 233,9223372036854775807,0x12c733349e00,0,DefaultEnhancedAscendMemoryPool,180355584,7484735488,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,232,kernel_graph_0_DataPrepareActor,adam_m.model.layers.1.feed_forward.w3.weight,kernel_graph_0_DataPrepareActor,{2295-2322-4580-4607-6870-6897-9160-9187-11450-11477-13740-13767-16030-16057-18320-18347-20610-20637-22900-22927},22927,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 234,9223372036854775807,0x12c73df4a000,0,DefaultEnhancedAscendMemoryPool,524288512,8009023488,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,233,kernel_graph_0_DataPrepareActor,adam_m.lm_head.weight,kernel_graph_0_DataPrepareActor,{2327-2354-4612-4639-6902-6929-9192-9219-11482-11509-13772-13799-16062-16089-18352-18379-20642-20669-22932-22959},22959,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 235,9223372036854775807,0x12c75d34a200,0,DefaultEnhancedAscendMemoryPool,16896,8009023488,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,234,kernel_graph_0_DataPrepareActor,adam_m.model.layers.0.ffn_norm.weight,kernel_graph_0_DataPrepareActor,{2359-2386-4644-4671-6934-6961-9224-9251-11514-11541-13804-13831-16094-16121-18384-18411-20674-20701-22964-22991},22991,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 236,9223372036854775807,0x12c75d34e400,0,DefaultEnhancedAscendMemoryPool,16896,8009023488,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,235,kernel_graph_0_DataPrepareActor,adam_m.model.layers.0.attention_norm.weight,kernel_graph_0_DataPrepareActor,{2391-2418-4676-4703-6966-6993-9256-9283-11546-11573-13836-13863-16126-16153-18416-18443-20706-20733-22996-23023},23023,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 237,9223372036854775807,0x12c75d352600,0,DefaultEnhancedAscendMemoryPool,16896,8009023488,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,236,kernel_graph_0_DataPrepareActor,adam_m.model.layers.1.ffn_norm.weight,kernel_graph_0_DataPrepareActor,{2423-2450-4708-4735-6998-7025-9288-9315-11578-11605-13868-13895-16158-16185-18448-18475-20738-20765-23028-23055},23055,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 238,9223372036854775807,0x12c75d356800,0,DefaultEnhancedAscendMemoryPool,16896,8009023488,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,237,kernel_graph_0_DataPrepareActor,adam_m.model.layers.1.attention_norm.weight,kernel_graph_0_DataPrepareActor,{2457-2482-4742-4767-7032-7057-9322-9347-11612-11637-13902-13927-16192-16217-18482-18507-20772-20797-23062-23087},23087,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 239,9223372036854775807,0x12c75d35aa00,0,DefaultEnhancedAscendMemoryPool,16896,8009023488,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Weight,238,kernel_graph_0_DataPrepareActor,adam_m.model.norm_out.weight,kernel_graph_0_DataPrepareActor,{2489-2515-4774-4800-7064-7090-9354-9380-11644-11670-13934-13960-16224-16250-18514-18540-20804-20830-23094-23120},23120,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 240,644,0x12c75d35ec00,0,DefaultEnhancedAscendMemoryPool,33280,8009023488,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Kernel,239,0_Default/network-LlamaForCausalLM/StridedSlice-op0,inputs0,0_Default/network-LlamaForCausalLM/StridedSlice-op0,{241-642},642,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,1,0, 241,1223,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,33280,8009023488,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,239,0_Default/network-LlamaForCausalLM/StridedSlice-op0,inputs0,0_Default/network-LlamaForCausalLM/StridedSlice-op0,{243-245-1220},1220,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 243,249,0x12c180008800,0,DefaultEnhancedAscendMemoryPool,134218240,8143241216,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,240,1_Default/network-LlamaForCausalLM/model-LlamaModel/tok_embeddings-LlamaEmbedding/Gather-op0,Default/network-LlamaForCausalLM/model-LlamaModel/tok_embeddings-LlamaEmbedding/Gather-op0,kernel_graph0,{247},247,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 245,255,0x12c188008a00,0,DefaultEnhancedAscendMemoryPool,8704,8143241216,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,241,2_Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/NotEqual-op0,Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/NotEqual-op0,kernel_graph0,{250-253},253,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 247,1201,0x12c18800ac00,0,DefaultEnhancedAscendMemoryPool,67109376,8210350080,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,3_Default/network-LlamaForCausalLM/model-LlamaModel/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/Cast-op0,kernel_graph0,{259-388-1199},1199,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 250,678,0x12c180008800,0,DefaultEnhancedAscendMemoryPool,33280,8210350080,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,4_Default/network-LlamaForCausalLM/Cast-op1,Default/network-LlamaForCausalLM/Cast-op1,kernel_graph0,{251-270-631-675},675,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 253,284,0x12c180010a00,0,DefaultEnhancedAscendMemoryPool,16896,8210350080,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,6_Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Cast-op0,kernel_graph0,{278-282},282,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 256,1127,0x12c180014c00,0,DefaultEnhancedAscendMemoryPool,524800,8210350080,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,7_Default/network-LlamaForCausalLM/model-LlamaModel/freqs_mgr-FreqsMgr/StridedSlice-op0,Default/network-LlamaForCausalLM/model-LlamaModel/freqs_mgr-FreqsMgr/StridedSlice-op0,kernel_graph0,{257-321-344-474-489-862-883-1104-1125},1125,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 259,265,0x12c18c00ae00,0,DefaultEnhancedAscendMemoryPool,134218240,8344567808,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,9_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op0,kernel_graph0,{263},263,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 261,269,0x12c19400b000,0,DefaultEnhancedAscendMemoryPool,134218240,8478785536,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,10_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{267},267,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 262,1208,0x12c180094e00,0,DefaultEnhancedAscendMemoryPool,33280,8478785536,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,10_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{1204},1204,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 263,266,0x12c18009d000,0,DefaultEnhancedAscendMemoryPool,16777728,8478785536,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,10_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 267,1179,0x12c18009d000,0,DefaultEnhancedAscendMemoryPool,67109376,8478785536,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,11_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op1,kernel_graph0,{292-297-312-335-1149-1163-1177},1177,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 270,278,0x12c18409d200,0,DefaultEnhancedAscendMemoryPool,512,8478785536,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,12_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/ReduceSum-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/ReduceSum-op0,kernel_graph0,{276},276,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 272,274,0x12c18409d400,0,DefaultEnhancedAscendMemoryPool,1024,8478785536,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,272,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 273,275,0x12c18409d800,0,DefaultEnhancedAscendMemoryPool,1024,8478785536,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,273,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 276,689,0x12c18409d400,0,DefaultEnhancedAscendMemoryPool,512,8478785536,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,13_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Add-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Add-op0,kernel_graph0,{619-686},686,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 280,285,0x12c18409d600,0,DefaultEnhancedAscendMemoryPool,2097664,8478785536,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,15_Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/ExpandDims-op0,Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/ExpandDims-op0,kernel_graph0,{282},282,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 282,288,0x12c18429d800,0,DefaultEnhancedAscendMemoryPool,16777728,8478785536,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,16_Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Mul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Mul-op0,kernel_graph0,{286},286,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 286,292,0x12c18529da00,0,DefaultEnhancedAscendMemoryPool,16777728,8478785536,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,17_Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Sub-op0,Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Sub-op0,kernel_graph0,{288-290},290,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 290,1085,0x12c18409d600,0,DefaultEnhancedAscendMemoryPool,8389120,8478785536,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,19_Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Cast-op1,kernel_graph0,{373-520-838-1079},1079,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 294,1173,0x12c18489d800,0,DefaultEnhancedAscendMemoryPool,33554944,8478785536,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,21_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/Cast-op0,kernel_graph0,{297-1170},1170,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 296,304,0x12c18c00ae00,0,DefaultEnhancedAscendMemoryPool,67109376,8478785536,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,22_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,kernel_graph0,{299-302},302,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 297,299,0x12c18689da00,0,DefaultEnhancedAscendMemoryPool,20972544,8478785536,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,22_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 301,308,0x12c19000b000,0,DefaultEnhancedAscendMemoryPool,67109376,8478785536,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,24_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,kernel_graph0,{306},306,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 302,305,0x12c180010a00,0,DefaultEnhancedAscendMemoryPool,2048,8478785536,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,24_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 306,325,0x12c19400b200,0,DefaultEnhancedAscendMemoryPool,134218240,8478785536,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,25_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op2,kernel_graph0,{321-323},323,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 309,1188,0x12c18c00ae00,0,DefaultEnhancedAscendMemoryPool,33554944,8478785536,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,26_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/Cast-op0,kernel_graph0,{312-1185},1185,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 311,319,0x12c18e00b000,0,DefaultEnhancedAscendMemoryPool,67109376,8478785536,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,27_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,kernel_graph0,{314-317},317,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 312,314,0x12c18689da00,0,DefaultEnhancedAscendMemoryPool,20972544,8478785536,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,27_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 316,328,0x12c19c00b400,0,DefaultEnhancedAscendMemoryPool,67109376,8545894400,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,29_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,kernel_graph0,{326},326,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 317,320,0x12c180010a00,0,DefaultEnhancedAscendMemoryPool,2048,8545894400,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,29_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 321,364,0x12c1a000b600,0,DefaultEnhancedAscendMemoryPool,134218240,8680112128,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,30_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op2,kernel_graph0,{362},362,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 323,361,0x12c1a800b800,0,DefaultEnhancedAscendMemoryPool,134218240,8814329856,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,31_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op1,kernel_graph0,{359},359,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 326,348,0x12c18e00b000,0,DefaultEnhancedAscendMemoryPool,134218240,8814329856,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,32_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op0,kernel_graph0,{344-346},346,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 329,1130,0x12c18689da00,0,DefaultEnhancedAscendMemoryPool,524800,8814329856,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,33_Default/network-LlamaForCausalLM/model-LlamaModel/freqs_mgr-FreqsMgr/StridedSlice-op1,Default/network-LlamaForCausalLM/model-LlamaModel/freqs_mgr-FreqsMgr/StridedSlice-op1,kernel_graph0,{330-349-359-479-494-864-885-1106-1128},1128,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 332,1159,0x12c19600b200,0,DefaultEnhancedAscendMemoryPool,33554944,8814329856,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,35_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/Cast-op0,kernel_graph0,{335-1156},1156,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 334,342,0x12c19800b400,0,DefaultEnhancedAscendMemoryPool,67109376,8814329856,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,36_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,kernel_graph0,{337-340},340,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 335,337,0x12c18691dc00,0,DefaultEnhancedAscendMemoryPool,20972544,8814329856,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,36_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 339,1083,0x12c1b000ba00,0,DefaultEnhancedAscendMemoryPool,67109376,8881438720,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,38_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,kernel_graph0,{373-1079},1079,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 340,343,0x12c180010a00,0,DefaultEnhancedAscendMemoryPool,2048,8881438720,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,38_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 344,354,0x12c19800b400,0,DefaultEnhancedAscendMemoryPool,134218240,8881438720,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,39_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op0,kernel_graph0,{352},352,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 346,351,0x12c1b400bc00,0,DefaultEnhancedAscendMemoryPool,134218240,9015656448,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,40_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op0,kernel_graph0,{349},349,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 349,355,0x12c18e00b000,0,DefaultEnhancedAscendMemoryPool,134218240,9015656448,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,41_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op1,kernel_graph0,{352},352,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 352,358,0x12c1b400bc00,0,DefaultEnhancedAscendMemoryPool,134218240,9015656448,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,42_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op0,kernel_graph0,{356},356,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 356,1081,0x12c18e00b000,0,DefaultEnhancedAscendMemoryPool,67109376,9015656448,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,43_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op1,kernel_graph0,{373-1079},1079,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 359,365,0x12c19800b400,0,DefaultEnhancedAscendMemoryPool,134218240,9015656448,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,44_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op3,kernel_graph0,{362},362,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 362,368,0x12c1a800b800,0,DefaultEnhancedAscendMemoryPool,134218240,9015656448,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,45_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op1,kernel_graph0,{366},366,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 366,1082,0x12c1b400bc00,0,DefaultEnhancedAscendMemoryPool,67109376,9015656448,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,46_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op3,kernel_graph0,{373-1079},1079,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 369,1086,0x12c18691dc00,0,DefaultEnhancedAscendMemoryPool,8389120,9015656448,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,47_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{1079},1079,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 370,1087,0x12c18711de00,0,DefaultEnhancedAscendMemoryPool,8389120,9015656448,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,47_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{1079},1079,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 371,1088,0x12c18409d200,0,DefaultEnhancedAscendMemoryPool,512,9015656448,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,47_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{1079},1079,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 372,1089,0x12c19800b400,0,DefaultEnhancedAscendMemoryPool,67109376,9015656448,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,47_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{377-1079},1079,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 373,375,0x12c19c00b600,0,DefaultEnhancedAscendMemoryPool,84936192,9015656448,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,47_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 376,1059,0x12c19c00b600,0,DefaultEnhancedAscendMemoryPool,67109376,9015656448,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,48_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,kernel_graph0,{379-384-1057},1057,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 377,379,0x12c180010a00,0,DefaultEnhancedAscendMemoryPool,2048,9015656448,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,48_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 381,1067,0x12c19200b200,0,DefaultEnhancedAscendMemoryPool,33554944,9015656448,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,50_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/Cast-op0,kernel_graph0,{384-1065},1065,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 383,390,0x12c1a000b800,0,DefaultEnhancedAscendMemoryPool,67109376,9015656448,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,51_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,kernel_graph0,{386-388},388,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 384,386,0x12c19400b400,0,DefaultEnhancedAscendMemoryPool,20972544,9015656448,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,51_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 388,1039,0x12c1a400ba00,0,DefaultEnhancedAscendMemoryPool,67109376,9015656448,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,53_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/Add-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/Add-op0,kernel_graph0,{391-429-1037},1037,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 391,397,0x12c1b800be00,0,DefaultEnhancedAscendMemoryPool,134218240,9082765312,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,54_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op0,kernel_graph0,{395},395,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 393,401,0x12c1c000c000,0,DefaultEnhancedAscendMemoryPool,134218240,9216983040,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,55_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{399},399,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 394,1046,0x12c18791e000,0,DefaultEnhancedAscendMemoryPool,33280,9216983040,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,55_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{1042},1042,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 395,398,0x12c19400b400,0,DefaultEnhancedAscendMemoryPool,16777728,9216983040,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,55_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 399,1018,0x12c1a000b800,0,DefaultEnhancedAscendMemoryPool,67109376,9216983040,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,56_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op1,kernel_graph0,{401-406-413-1002-1016},1016,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 403,1012,0x12c1a800bc00,0,DefaultEnhancedAscendMemoryPool,90178048,9216983040,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,58_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/Cast-op0,kernel_graph0,{406-1009},1009,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 405,994,0x12c1b800be00,0,DefaultEnhancedAscendMemoryPool,180355584,9216983040,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,59_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,kernel_graph0,{408-419-992},992,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 406,408,0x12c19400b400,0,DefaultEnhancedAscendMemoryPool,20972544,9216983040,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,59_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 410,1027,0x12c1c2c0c000,0,DefaultEnhancedAscendMemoryPool,90178048,9219080192,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,61_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/Cast-op0,kernel_graph0,{413-1024},1024,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 412,1000,0x12c1c820c200,0,DefaultEnhancedAscendMemoryPool,180355584,9399435264,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,62_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,kernel_graph0,{416-997},997,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 413,415,0x12c19400b400,0,DefaultEnhancedAscendMemoryPool,20972544,9399435264,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,62_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 416,990,0x12c1d2e0c400,0,DefaultEnhancedAscendMemoryPool,180355584,9579790336,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,63_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/SiLU-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/SiLU-op0,kernel_graph0,{417-419-988},988,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 419,977,0x12c1dda0c600,0,DefaultEnhancedAscendMemoryPool,180355584,9760145408,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,65_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/Mul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/Mul-op0,kernel_graph0,{420-425-975},975,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 422,985,0x12c1e860c800,0,DefaultEnhancedAscendMemoryPool,90178048,9850322944,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,67_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/Cast-op0,kernel_graph0,{425-983},983,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 424,431,0x12c1edc0ca00,0,DefaultEnhancedAscendMemoryPool,67109376,9917431808,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,68_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,kernel_graph0,{427-429},429,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 425,427,0x12c19400b400,0,DefaultEnhancedAscendMemoryPool,20972544,9917431808,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,68_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 429,957,0x12c1f1c0cc00,0,DefaultEnhancedAscendMemoryPool,67109376,9984540672,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,70_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/Add-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/Add-op1,kernel_graph0,{432-535-955},955,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 432,438,0x12c1f5c0ce00,0,DefaultEnhancedAscendMemoryPool,134218240,10118758400,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,71_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op0,kernel_graph0,{436},436,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 434,442,0x12c1fdc0d000,0,DefaultEnhancedAscendMemoryPool,134218240,10252976128,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,72_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{440},440,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 435,964,0x12c187926200,0,DefaultEnhancedAscendMemoryPool,33280,10252976128,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,72_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{960},960,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 436,439,0x12c19400b400,0,DefaultEnhancedAscendMemoryPool,16777728,10252976128,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,72_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 440,935,0x12c1edc0ca00,0,DefaultEnhancedAscendMemoryPool,67109376,10252976128,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,73_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op1,kernel_graph0,{442-447-462-507-905-919-933},933,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 444,929,0x12c1ad60be00,0,DefaultEnhancedAscendMemoryPool,33554944,10252976128,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,75_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/Cast-op0,kernel_graph0,{447-926},926,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 446,454,0x12c1f5c0ce00,0,DefaultEnhancedAscendMemoryPool,67109376,10252976128,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,76_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,kernel_graph0,{449-452},452,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 447,449,0x12c19400b400,0,DefaultEnhancedAscendMemoryPool,20972544,10252976128,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,76_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 451,458,0x12c1f9c0d000,0,DefaultEnhancedAscendMemoryPool,67109376,10252976128,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,78_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,kernel_graph0,{456},456,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 452,455,0x12c180010a00,0,DefaultEnhancedAscendMemoryPool,2048,10252976128,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,78_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 456,478,0x12c1fdc0d200,0,DefaultEnhancedAscendMemoryPool,134218240,10252976128,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,79_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op2,kernel_graph0,{474-476},476,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 459,944,0x12c1f5c0ce00,0,DefaultEnhancedAscendMemoryPool,33554944,10252976128,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,80_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/Cast-op0,kernel_graph0,{462-941},941,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 461,469,0x12c1f7c0d000,0,DefaultEnhancedAscendMemoryPool,67109376,10252976128,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,81_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,kernel_graph0,{464-467},467,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 462,464,0x12c19400b400,0,DefaultEnhancedAscendMemoryPool,20972544,10252976128,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,81_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 466,473,0x12c205c0d400,0,DefaultEnhancedAscendMemoryPool,67109376,10320084992,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,83_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,kernel_graph0,{471},471,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 467,470,0x12c180010a00,0,DefaultEnhancedAscendMemoryPool,2048,10320084992,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,83_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 471,493,0x12c209c0d600,0,DefaultEnhancedAscendMemoryPool,134218240,10454302720,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,84_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op0,kernel_graph0,{489-491},491,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 474,484,0x12c211c0d800,0,DefaultEnhancedAscendMemoryPool,134218240,10588520448,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,85_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op2,kernel_graph0,{482},482,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 476,481,0x12c219c0da00,0,DefaultEnhancedAscendMemoryPool,134218240,10722738176,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,86_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op1,kernel_graph0,{479},479,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 479,485,0x12c1f7c0d000,0,DefaultEnhancedAscendMemoryPool,134218240,10722738176,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,87_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op3,kernel_graph0,{482},482,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 482,488,0x12c219c0da00,0,DefaultEnhancedAscendMemoryPool,134218240,10722738176,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,88_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op1,kernel_graph0,{486},486,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 486,841,0x12c211c0d800,0,DefaultEnhancedAscendMemoryPool,67109376,10722738176,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,89_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op3,kernel_graph0,{520-838},838,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 489,499,0x12c215c0da00,0,DefaultEnhancedAscendMemoryPool,134218240,10722738176,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,90_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op0,kernel_graph0,{497},497,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 491,496,0x12c1f7c0d000,0,DefaultEnhancedAscendMemoryPool,134218240,10722738176,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,91_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op0,kernel_graph0,{494},494,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 494,500,0x12c1ffc0d200,0,DefaultEnhancedAscendMemoryPool,134218240,10722738176,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,92_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op1,kernel_graph0,{497},497,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 497,503,0x12c1f7c0d000,0,DefaultEnhancedAscendMemoryPool,134218240,10722738176,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,93_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op0,kernel_graph0,{501},501,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 501,840,0x12c215c0da00,0,DefaultEnhancedAscendMemoryPool,67109376,10722738176,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,94_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op1,kernel_graph0,{520-838},838,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 504,915,0x12c219c0dc00,0,DefaultEnhancedAscendMemoryPool,33554944,10722738176,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,95_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/Cast-op0,kernel_graph0,{507-912},912,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 506,514,0x12c21bc0de00,0,DefaultEnhancedAscendMemoryPool,67109376,10722738176,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,96_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,kernel_graph0,{509-512},512,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 507,509,0x12c21fc0e000,0,DefaultEnhancedAscendMemoryPool,20972544,10722738176,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,96_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 511,842,0x12c1f7c0d000,0,DefaultEnhancedAscendMemoryPool,67109376,10722738176,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,98_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,kernel_graph0,{520-838},838,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 512,515,0x12c180010a00,0,DefaultEnhancedAscendMemoryPool,2048,10722738176,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,98_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 516,844,0x12c1af60c000,0,DefaultEnhancedAscendMemoryPool,8389120,10722738176,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,99_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{838},838,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 517,845,0x12c19400b400,0,DefaultEnhancedAscendMemoryPool,8389120,10722738176,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,99_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{838},838,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 518,846,0x12c180010a00,0,DefaultEnhancedAscendMemoryPool,512,10722738176,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,99_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{838},838,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 519,847,0x12c21bc0de00,0,DefaultEnhancedAscendMemoryPool,67109376,10722738176,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,99_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{524-838},838,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 520,522,0x12c1fbc0d200,0,DefaultEnhancedAscendMemoryPool,84936192,10722738176,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,99_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 523,818,0x12c1fbc0d200,0,DefaultEnhancedAscendMemoryPool,67109376,10722738176,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,100_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,kernel_graph0,{526-531-816},816,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 524,526,0x12c180010c00,0,DefaultEnhancedAscendMemoryPool,2048,10722738176,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,100_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 528,826,0x12c1ffc0d400,0,DefaultEnhancedAscendMemoryPool,33554944,10722738176,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,102_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/Cast-op0,kernel_graph0,{531-824},824,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 530,537,0x12c201c0d600,0,DefaultEnhancedAscendMemoryPool,67109376,10722738176,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,103_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,kernel_graph0,{533-535},535,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 531,533,0x12c19480b600,0,DefaultEnhancedAscendMemoryPool,20972544,10722738176,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,103_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 535,793,0x12c205c0d800,0,DefaultEnhancedAscendMemoryPool,67109376,10722738176,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,105_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/Add-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/Add-op0,kernel_graph0,{538-576-791},791,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 538,544,0x12c21fc0e000,0,DefaultEnhancedAscendMemoryPool,134218240,10823401472,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,106_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op0,kernel_graph0,{542},542,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 540,548,0x12c227c0e200,0,DefaultEnhancedAscendMemoryPool,134218240,10957619200,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,107_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{546},546,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 541,800,0x12c1afe0c200,0,DefaultEnhancedAscendMemoryPool,33280,10957619200,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,107_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{796},796,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 542,545,0x12c19480b600,0,DefaultEnhancedAscendMemoryPool,16777728,10957619200,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,107_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 546,772,0x12c201c0d600,0,DefaultEnhancedAscendMemoryPool,67109376,10957619200,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,108_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op1,kernel_graph0,{548-553-560-756-770},770,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 550,766,0x12c209c0da00,0,DefaultEnhancedAscendMemoryPool,90178048,10957619200,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,110_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/Cast-op0,kernel_graph0,{553-763},763,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 552,748,0x12c21fc0e000,0,DefaultEnhancedAscendMemoryPool,180355584,10957619200,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,111_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,kernel_graph0,{555-566-746},746,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 553,555,0x12c19480b600,0,DefaultEnhancedAscendMemoryPool,20972544,10957619200,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,111_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 557,781,0x12c22a80e200,0,DefaultEnhancedAscendMemoryPool,90178048,10959716352,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,113_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/Cast-op0,kernel_graph0,{560-778},778,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 559,754,0x12c22fe0e400,0,DefaultEnhancedAscendMemoryPool,180355584,11140071424,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,114_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,kernel_graph0,{563-751},751,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 560,562,0x12c19480b600,0,DefaultEnhancedAscendMemoryPool,20972544,11140071424,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,114_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 563,744,0x12c23aa0e600,0,DefaultEnhancedAscendMemoryPool,180355584,11320426496,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,115_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/SiLU-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/SiLU-op0,kernel_graph0,{564-566-742},742,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 566,731,0x12c24560e800,0,DefaultEnhancedAscendMemoryPool,180355584,11500781568,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,117_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/Mul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/Mul-op0,kernel_graph0,{567-572-729},729,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 569,739,0x12c25020ea00,0,DefaultEnhancedAscendMemoryPool,90178048,11590959104,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,119_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/Cast-op0,kernel_graph0,{572-737},737,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 571,578,0x12c25580ec00,0,DefaultEnhancedAscendMemoryPool,67109376,11658067968,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,120_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,kernel_graph0,{574-576},576,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 572,574,0x12c19480b600,0,DefaultEnhancedAscendMemoryPool,20972544,11658067968,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,120_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 576,710,0x12c25980ee00,0,DefaultEnhancedAscendMemoryPool,67109376,11725176832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,122_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/Add-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/Add-op1,kernel_graph0,{579-708},708,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 579,585,0x12c25d80f000,0,DefaultEnhancedAscendMemoryPool,134218240,11859394560,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,123_Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Cast-op0,kernel_graph0,{583},583,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 581,589,0x12c26580f200,0,DefaultEnhancedAscendMemoryPool,134218240,11993612288,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,124_Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{587},587,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 582,717,0x12c1afe14400,0,DefaultEnhancedAscendMemoryPool,33280,11993612288,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,124_Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{713},713,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 583,586,0x12c19480b600,0,DefaultEnhancedAscendMemoryPool,16777728,11993612288,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,124_Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 587,693,0x12c25580ec00,0,DefaultEnhancedAscendMemoryPool,67109376,11993612288,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,125_Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Cast-op1,kernel_graph0,{589-594-691},691,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 591,702,0x12c25d80f000,0,DefaultEnhancedAscendMemoryPool,262144512,11993612288,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,127_Default/network-LlamaForCausalLM/lm_head-Linear/Cast-op0,Default/network-LlamaForCausalLM/lm_head-Linear/Cast-op0,kernel_graph0,{594-699},699,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 593,599,0x12c26d20f200,0,DefaultEnhancedAscendMemoryPool,524288512,12511608832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,128_Default/network-LlamaForCausalLM/lm_head-Linear/MatMul-op0,Default/network-LlamaForCausalLM/lm_head-Linear/MatMul-op0,kernel_graph0,{597},597,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 594,596,0x12c19480b600,0,DefaultEnhancedAscendMemoryPool,20972544,12511608832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,128_Default/network-LlamaForCausalLM/lm_head-Linear/MatMul-op0,Default/network-LlamaForCausalLM/lm_head-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 597,606,0x12c28c60f400,0,DefaultEnhancedAscendMemoryPool,1048576512,13560184832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,129_Default/network-LlamaForCausalLM/Cast-op0,Default/network-LlamaForCausalLM/Cast-op0,kernel_graph0,{601-604},604,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 600,607,0x12c1afe1c600,0,DefaultEnhancedAscendMemoryPool,33280,13560184832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,130_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/ReduceMax-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/ReduceMax-op0,kernel_graph0,{604},604,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 601,603,0x12c180010c00,0,DefaultEnhancedAscendMemoryPool,1536,13560184832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,130_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/ReduceMax-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/ReduceMax-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 604,640,0x12c2cae0f600,0,DefaultEnhancedAscendMemoryPool,1048576512,14608760832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,131_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Sub-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Sub-op0,kernel_graph0,{610-638},638,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 608,621,0x12c180010c00,0,DefaultEnhancedAscendMemoryPool,512,14608760832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,132_Default/Mul-op0,Default/Mul-op0,kernel_graph0,{619},619,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 610,618,0x12c26d20f200,0,DefaultEnhancedAscendMemoryPool,1048576512,14608760832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,133_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Exp-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Exp-op0,kernel_graph0,{612},612,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 612,637,0x12c1afe1c600,0,DefaultEnhancedAscendMemoryPool,33280,14608760832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,134_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/ReduceSum-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/ReduceSum-op0,kernel_graph0,{635},635,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 614,616,0x12c180010e00,0,DefaultEnhancedAscendMemoryPool,1024,14608760832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,614,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 615,617,0x12c180011200,0,DefaultEnhancedAscendMemoryPool,1024,14608760832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,615,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 619,625,0x12c180010e00,0,DefaultEnhancedAscendMemoryPool,512,14608760832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,135_Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_RealDiv/RealDiv-op0,Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_RealDiv/RealDiv-op0,kernel_graph0,{623},623,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 622,630,0x12c180010c00,0,DefaultEnhancedAscendMemoryPool,512,14608760832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,136_Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_RealDiv/SumExt-op0,Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_RealDiv/SumExt-op0,kernel_graph0,{626-628},628,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 623,626,0x12c180011000,0,DefaultEnhancedAscendMemoryPool,1536,14608760832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,136_Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_RealDiv/SumExt-op0,Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_RealDiv/SumExt-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 628,633,0x12c1afe24800,0,DefaultEnhancedAscendMemoryPool,33280,14608760832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,138_Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_ReduceSum/Tile-op0,Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_ReduceSum/Tile-op0,kernel_graph0,{631},631,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 631,657,0x12c1afe2ca00,0,DefaultEnhancedAscendMemoryPool,33280,14608760832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,139_Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_Mul/Mul-op0,Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_Mul/Mul-op0,kernel_graph0,{633-654},654,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 635,641,0x12c1afe24800,0,DefaultEnhancedAscendMemoryPool,33280,14608760832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,141_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Log-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Log-op0,kernel_graph0,{638},638,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 638,663,0x12c26d20f200,0,DefaultEnhancedAscendMemoryPool,1048576512,14608760832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,142_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Sub-op1,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Sub-op1,kernel_graph0,{649-661},661,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 642,648,0x12c1afe1c600,0,DefaultEnhancedAscendMemoryPool,33280,14608760832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,143_Default/network-LlamaForCausalLM/StridedSlice-op1,Default/network-LlamaForCausalLM/StridedSlice-op1,kernel_graph0,{644-646},646,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 646,664,0x12c2aba0f400,0,DefaultEnhancedAscendMemoryPool,1048576512,14608760832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,145_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/OneHot-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/OneHot-op0,kernel_graph0,{651-661},661,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 649,653,0x12c2ea20f600,0,DefaultEnhancedAscendMemoryPool,1048576512,15133048832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,146_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Exp-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Exp-op0,kernel_graph0,{651},651,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 651,656,0x12c328a0f800,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,147_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Sub-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Sub-op0,kernel_graph0,{654},654,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 654,660,0x12c2ea20f600,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,148_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Mul-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Mul-op0,kernel_graph0,{658},658,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 658,701,0x12c328a0f800,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,149_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op0,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op0,kernel_graph0,{691-699},699,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 661,667,0x12c2ea20f600,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,150_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Mul-op1,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Mul-op1,kernel_graph0,{665},665,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 665,674,0x12c26d20f200,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,151_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Neg-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Neg-op0,kernel_graph0,{668},668,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 668,677,0x12c1afe1c600,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,152_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/ReduceSum-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/ReduceSum-op0,kernel_graph0,{675},675,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 670,672,0x12c180010c00,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,670,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 671,673,0x12c180011000,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,671,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 675,685,0x12c1afe24800,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,153_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Mul-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Mul-op0,kernel_graph0,{679},679,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 679,688,0x12c180010c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,154_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/ReduceSum-op1,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/ReduceSum-op1,kernel_graph0,{686},686,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 681,683,0x12c180010e00,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,681,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 682,684,0x12c180011200,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,682,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 686,2522,0x12c180010e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,155_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/RealDiv-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/RealDiv-op0,kernel_graph0,{2520},2520,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 690,697,0x12c347e0fa00,0,DefaultEnhancedAscendMemoryPool,262144512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,156_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op29,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op29,kernel_graph0,{695},695,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 691,694,0x12c19480b600,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,156_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op29,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op29,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 695,1276,0x12c26d20f200,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,157_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op29,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op29,kernel_graph0,{1274},1274,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 698,707,0x12c25580ec00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,158_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op0,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op0,kernel_graph0,{703-705},705,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 699,703,0x12c347e0fa00,0,DefaultEnhancedAscendMemoryPool,155190272,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,158_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op0,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 705,715,0x12c25d80f000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,160_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op1,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op1,kernel_graph0,{713},713,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 708,716,0x12c28c60f400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,161_Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Cast-op2,kernel_graph0,{713},713,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 711,721,0x12c25580ec00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,162_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op0,kernel_graph0,{719},719,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 712,1291,0x12c180008800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,162_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op0,kernel_graph0,{1289},1289,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 713,718,0x12c19480b600,0,DefaultEnhancedAscendMemoryPool,16778752,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,162_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 719,808,0x12c25d80ee00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,163_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op0,kernel_graph0,{723-729-737-805},805,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 722,727,0x12c180010c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,164_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Mul-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Mul-op0,kernel_graph0,{725},725,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 725,1491,0x12c18409d400,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,166_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Maximum-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Maximum-op0,kernel_graph0,{1489},1489,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 728,735,0x12c25580ec00,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,167_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op27,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op27,kernel_graph0,{733},733,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 729,732,0x12c19480b600,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,167_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op27,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op27,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 733,1270,0x12c24560e800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,168_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op27,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op27,kernel_graph0,{1268},1268,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 736,749,0x12c26180f000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,169_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op1,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op1,kernel_graph0,{740-742-746},746,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 737,740,0x12c19480b600,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,169_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op1,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 742,765,0x12c25020ea00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,171_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op0,kernel_graph0,{744-756-763},763,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 746,753,0x12c23aa0e600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,173_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op1,kernel_graph0,{749-751},751,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 751,780,0x12c21fc0e000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,175_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/Grad_SiLU/SiLUGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/Grad_SiLU/SiLUGrad-op0,kernel_graph0,{770-778},778,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 755,761,0x12c26180f000,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,176_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op28,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op28,kernel_graph0,{759},759,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 756,758,0x12c19480b600,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,176_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op28,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op28,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 759,1273,0x12c22fe0e400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,177_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op28,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op28,kernel_graph0,{1271},1271,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 762,786,0x12c23aa0e600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,178_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op2,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op2,kernel_graph0,{767-784},784,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 763,767,0x12c19480b600,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,178_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op2,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 769,776,0x12c23ea0e800,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,180_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op26,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op26,kernel_graph0,{774},774,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 770,773,0x12c24400ea00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,180_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op26,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op26,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 774,1267,0x12c26180f000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,181_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op26,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op26,kernel_graph0,{1265},1265,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 777,787,0x12c201c0d600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,182_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op3,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op3,kernel_graph0,{782-784},784,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 778,782,0x12c19480b600,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,182_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op3,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op3,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 784,790,0x12c23ea0e800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,184_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/AddN-op0,kernel_graph0,{788},788,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 788,798,0x12c25020ea00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,185_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op2,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op2,kernel_graph0,{796},796,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 791,799,0x12c23aa0e600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,186_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op2,kernel_graph0,{796},796,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 794,804,0x12c201c0d600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,187_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op1,kernel_graph0,{802},802,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 795,1285,0x12c1afe14400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,187_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op1,kernel_graph0,{1283},1283,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 796,801,0x12c19480b600,0,DefaultEnhancedAscendMemoryPool,16778752,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,187_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 802,807,0x12c209c0d800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,188_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op1,kernel_graph0,{805},805,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 805,972,0x12c201c0d600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,189_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/AddN-op0,kernel_graph0,{810-816-824-969},969,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 809,814,0x12c180010c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,190_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Sub-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Sub-op0,kernel_graph0,{812},812,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 812,1514,0x12c180011000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,192_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/LessEqual-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/LessEqual-op0,kernel_graph0,{1494-1512},1512,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 815,822,0x12c23aa0e600,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,193_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op25,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op25,kernel_graph0,{820},820,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 816,819,0x12c19480b600,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,193_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op25,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op25,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 820,1264,0x12c1fbc0d200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,194_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op25,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op25,kernel_graph0,{1262},1262,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 823,832,0x12c23aa0e600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,195_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op4,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op4,kernel_graph0,{827-830},830,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 824,827,0x12c19480b600,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,195_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op4,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op4,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 829,843,0x12c23ea0e800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,197_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op0,kernel_graph0,{838},838,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 830,833,0x12c180011200,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,197_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 834,855,0x12c23aa0e600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,198_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,kernel_graph0,{853},853,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 835,852,0x12c205c0d800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,198_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,kernel_graph0,{850},850,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 836,859,0x12c209c0da00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,198_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,kernel_graph0,{857},857,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 837,848,0x12c180010c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,198_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 838,849,0x12c28c60f400,0,DefaultEnhancedAscendMemoryPool,515969024,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,198_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 850,866,0x12c20dc0dc00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,199_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op5,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op5,kernel_graph0,{862-864},864,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 853,887,0x12c25020ea00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,200_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op3,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op3,kernel_graph0,{883-885},885,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 856,914,0x12c1f7c0d000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,201_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op3,kernel_graph0,{860-905-912},912,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 857,860,0x12c180011200,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,201_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op3,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 862,872,0x12c205c0d800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,203_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op4,kernel_graph0,{870},870,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 864,869,0x12c25820ec00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,204_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op5,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op5,kernel_graph0,{867},867,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 867,873,0x12c23aa0e600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,205_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op1,kernel_graph0,{870},870,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 870,876,0x12c25820ec00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,206_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op1,kernel_graph0,{874},874,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 874,880,0x12c23aa0e600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,207_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op6,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op6,kernel_graph0,{878},878,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 877,928,0x12c23ea0e800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,208_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op2,kernel_graph0,{881-919-926},926,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 878,881,0x12c180011200,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,208_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 883,893,0x12c25820ec00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,210_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op2,kernel_graph0,{891},891,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 885,890,0x12c205c0d800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,211_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op3,kernel_graph0,{888},888,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 888,894,0x12c25020ea00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,212_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op0,kernel_graph0,{891},891,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 891,897,0x12c205c0d800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,213_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op0,kernel_graph0,{895},895,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 895,901,0x12c23aa0e600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,214_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op4,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op4,kernel_graph0,{899},899,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 898,943,0x12c25020ea00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,215_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op1,kernel_graph0,{902-933-941},941,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 899,902,0x12c180011200,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,215_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 904,910,0x12c1ffc0d400,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,217_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op24,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op24,kernel_graph0,{908},908,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 905,907,0x12c19400b400,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,217_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op24,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op24,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 908,1261,0x12c23aa0e600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,218_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op24,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op24,kernel_graph0,{1259},1259,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 911,951,0x12c25420ec00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,219_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op7,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op7,kernel_graph0,{916-947},947,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 912,916,0x12c19400b400,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,219_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op7,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op7,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 918,924,0x12c1ffc0d400,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,221_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op23,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op23,kernel_graph0,{922},922,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 919,921,0x12c19400b400,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,221_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op23,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op23,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 922,1258,0x12c1f7c0d000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,222_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op23,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op23,kernel_graph0,{1256},1256,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 925,950,0x12c25820ee00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,223_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op6,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op6,kernel_graph0,{930-947},947,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 926,930,0x12c19400b400,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,223_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op6,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op6,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 932,939,0x12c1ffc0d400,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,225_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op22,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op22,kernel_graph0,{937},937,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 933,936,0x12c19400b400,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,225_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op22,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op22,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 937,1255,0x12c1edc0ca00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,226_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op22,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op22,kernel_graph0,{1253},1253,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 940,949,0x12c25c20f000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,227_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op5,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op5,kernel_graph0,{945-947},947,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 941,945,0x12c26020f200,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,227_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op5,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op5,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 947,954,0x12c25020ea00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,229_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/AddN-op0,kernel_graph0,{952},952,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 952,962,0x12c25420ec00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,230_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op7,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op7,kernel_graph0,{960},960,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 955,963,0x12c205c0d800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,231_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op2,kernel_graph0,{960},960,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 958,968,0x12c20dc0da00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,232_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op2,kernel_graph0,{966},966,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 959,1288,0x12c18000ca00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,232_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op2,kernel_graph0,{1286},1286,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 960,965,0x12c19400b400,0,DefaultEnhancedAscendMemoryPool,16778752,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,232_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 966,971,0x12c1f1c0cc00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,233_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op2,kernel_graph0,{969},969,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 969,1054,0x12c23ea0e800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,234_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/AddN-op0,kernel_graph0,{972-975-983-1051},1051,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 974,981,0x12c1f1c0cc00,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,236_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op20,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op20,kernel_graph0,{979},979,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 975,978,0x12c19400b400,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,236_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op20,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op20,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 979,1249,0x12c1dda0c600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,237_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op20,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op20,kernel_graph0,{1247},1247,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 982,995,0x12c25020ea00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,238_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op8,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op8,kernel_graph0,{986-988-992},992,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 983,986,0x12c19400b400,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,238_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op8,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op8,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 988,1011,0x12c1ffc0d400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,240_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op6,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op6,kernel_graph0,{990-1002-1009},1009,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 992,999,0x12c1d2e0c400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,242_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op7,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op7,kernel_graph0,{995-997},997,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 997,1026,0x12c1b800be00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,244_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/Grad_SiLU/SiLUGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/Grad_SiLU/SiLUGrad-op1,kernel_graph0,{1016-1024},1024,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1001,1007,0x12c1e860c800,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,245_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op21,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op21,kernel_graph0,{1005},1005,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1002,1004,0x12c19400b400,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,245_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op21,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op21,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1005,1252,0x12c25020ea00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,246_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op21,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op21,kernel_graph0,{1250},1250,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1008,1032,0x12c1e860c800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,247_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op9,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op9,kernel_graph0,{1013-1030},1030,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1009,1013,0x12c1ec60ca00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,247_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op9,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op9,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1015,1022,0x12c1f1c0cc00,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,249_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op19,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op19,kernel_graph0,{1020},1020,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1016,1019,0x12c1ec60ca00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,249_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op19,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op19,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1020,1246,0x12c1c820c200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,250_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op19,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op19,kernel_graph0,{1244},1244,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1023,1033,0x12c1a000b800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,251_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op10,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op10,kernel_graph0,{1028-1030},1030,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1024,1028,0x12c1ec60ca00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,251_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op10,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op10,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1030,1036,0x12c1f1c0cc00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,253_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/AddN-op0,kernel_graph0,{1034},1034,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1034,1044,0x12c1d2e0c400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,254_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op8,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op8,kernel_graph0,{1042},1042,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1037,1045,0x12c1b800be00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,255_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op2,kernel_graph0,{1042},1042,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1040,1050,0x12c1c000c000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,256_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op3,kernel_graph0,{1048},1048,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1041,1279,0x12c1afe18600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,256_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op3,kernel_graph0,{1277},1277,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1042,1047,0x12c19400b400,0,DefaultEnhancedAscendMemoryPool,16778752,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,256_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op3,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1048,1053,0x12c1e860c800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,257_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op3,kernel_graph0,{1051},1051,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1051,1216,0x12c1f1c0cc00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,258_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/AddN-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/AddN-op1,kernel_graph0,{1054-1057-1065-1213},1213,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1056,1063,0x12c1f5c0ce00,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,260_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op18,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op18,kernel_graph0,{1061},1061,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1057,1060,0x12c19400b400,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,260_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op18,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op18,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1061,1243,0x12c1e860c800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,261_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op18,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op18,kernel_graph0,{1241},1241,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1064,1073,0x12c25ae0ec00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,262_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op11,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op11,kernel_graph0,{1068-1071},1071,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1065,1068,0x12c1ec60ca00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,262_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op11,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op11,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1070,1084,0x12c23ea0e800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,264_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op4,kernel_graph0,{1079},1079,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1071,1074,0x12c180011200,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,264_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op4,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1075,1097,0x12c25ae0ec00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,265_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,kernel_graph0,{1095},1095,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1076,1094,0x12c1d2e0c400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,265_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,kernel_graph0,{1092},1092,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1077,1101,0x12c1d6e0c600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,265_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,kernel_graph0,{1099},1099,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1078,1090,0x12c180010c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,265_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1079,1091,0x12c1ffc0d400,0,DefaultEnhancedAscendMemoryPool,515969024,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,265_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1092,1108,0x12c18e00b000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,266_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op11,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op11,kernel_graph0,{1104-1106},1106,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1095,1131,0x12c19800b400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,267_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op9,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op9,kernel_graph0,{1125-1128},1128,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1098,1158,0x12c1d2e0c400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,268_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op7,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op7,kernel_graph0,{1102-1149-1156},1156,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1099,1102,0x12c180011200,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,268_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op7,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op7,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1104,1114,0x12c1a000b600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,270_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op10,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op10,kernel_graph0,{1112},1112,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1106,1111,0x12c1afe1c800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,271_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op11,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op11,kernel_graph0,{1109},1109,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1109,1115,0x12c18e00b000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,272_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op3,kernel_graph0,{1112},1112,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1112,1118,0x12c1afe1c800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,273_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op1,kernel_graph0,{1116},1116,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1116,1122,0x12c25ae0ec00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,274_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op12,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op12,kernel_graph0,{1120},1120,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1119,1172,0x12c1d6e0c600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,275_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op6,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op6,kernel_graph0,{1123-1163-1170},1170,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1120,1123,0x12c180011200,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,275_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op6,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op6,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1125,1137,0x12c18e00b000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,277_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op8,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op8,kernel_graph0,{1135},1135,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1128,1134,0x12c1a000b600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,278_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op9,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op9,kernel_graph0,{1132},1132,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1132,1138,0x12c19800b400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,279_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op2,kernel_graph0,{1135},1135,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1135,1141,0x12c1a000b600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,280_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op0,kernel_graph0,{1139},1139,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1139,1145,0x12c25ae0ec00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,281_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op10,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op10,kernel_graph0,{1143},1143,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1142,1187,0x12c23ea0e800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,282_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op5,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op5,kernel_graph0,{1146-1177-1185},1185,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1143,1146,0x12c180011200,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,282_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op5,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op5,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1148,1154,0x12c1f5c0ce00,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,284_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op17,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op17,kernel_graph0,{1152},1152,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1149,1151,0x12c1ec60ca00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,284_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op17,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op17,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1152,1240,0x12c25ae0ec00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,285_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op17,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op17,kernel_graph0,{1238},1238,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1155,1195,0x12c18e00b000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,286_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op14,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op14,kernel_graph0,{1160-1191},1191,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1156,1160,0x12c1ec60ca00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,286_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op14,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op14,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1162,1168,0x12c1f5c0ce00,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,288_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op16,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op16,kernel_graph0,{1166},1166,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1163,1165,0x12c1ec60ca00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,288_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op16,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op16,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1166,1237,0x12c1d2e0c400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,289_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op16,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op16,kernel_graph0,{1235},1235,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1169,1194,0x12c1afe1c800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,290_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op13,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op13,kernel_graph0,{1174-1191},1191,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1170,1174,0x12c1ec60ca00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,290_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op13,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op13,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1176,1183,0x12c1f5c0ce00,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,292_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op15,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op15,kernel_graph0,{1181},1181,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1177,1180,0x12c1ec60ca00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,292_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op15,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op15,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1181,1234,0x12c18009d000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,293_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op15,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op15,kernel_graph0,{1232},1232,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1184,1193,0x12c1d6e0c600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,294_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op12,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op12,kernel_graph0,{1189-1191},1191,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1185,1189,0x12c1ec60ca00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,294_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op12,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op12,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1191,1198,0x12c23ea0e800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,296_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/AddN-op0,kernel_graph0,{1196},1196,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1196,1206,0x12c1afe1c800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,297_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op13,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op13,kernel_graph0,{1204},1204,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1199,1207,0x12c1b7e1ca00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,298_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op2,kernel_graph0,{1204},1204,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1202,1212,0x12c1bfe1cc00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,299_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op4,kernel_graph0,{1210},1210,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1203,1282,0x12c180011200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,299_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op4,kernel_graph0,{1280},1280,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1204,1209,0x12c1ec60ca00,0,DefaultEnhancedAscendMemoryPool,16778752,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,299_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op4,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1210,1215,0x12c1d6e0c600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,300_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op4,kernel_graph0,{1213},1213,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1213,1219,0x12c23ea0e800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,301_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/AddN-op0,kernel_graph0,{1217},1217,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1217,1222,0x12c1afe1c800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,302_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op14,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op14,kernel_graph0,{1220},1220,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1220,1231,0x12c18409d600,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,303_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/tok_embeddings-LlamaEmbedding/Grad_Gather/UnsortedSegmentSum-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/tok_embeddings-LlamaEmbedding/Grad_Gather/UnsortedSegmentSum-op0,kernel_graph0,{1229},1229,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1224,1228,0x12c180010c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,304_Default/Mul-op1,Default/Mul-op1,kernel_graph0,{1226},1226,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1226,1292,0x12c18409d200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,305_Default/Reciprocal-op0,Default/Reciprocal-op0,kernel_graph0,{1229-1232-1235-1238-1241-1244-1247-1250-1253-1256-1259-1262-1265-1268-1271-1274-1277-1280-1283-1286-1289},1289,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1229,1527,0x12c1ffc0d400,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,306_Default/Mul-op2,Default/Mul-op2,kernel_graph0,{1293-1453-1525},1525,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1232,1530,0x12c1f1c0cc00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,307_Default/Mul-op3,Default/Mul-op3,kernel_graph0,{1301-1453-1528},1528,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1235,1533,0x12c180015400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,308_Default/Mul-op4,Default/Mul-op4,kernel_graph0,{1309-1453-1531},1531,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1238,1536,0x12c23ea0e800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,309_Default/Mul-op5,Default/Mul-op5,kernel_graph0,{1317-1453-1534},1534,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1241,1539,0x12c25ae0ec00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,310_Default/Mul-op6,Default/Mul-op6,kernel_graph0,{1325-1453-1537},1537,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1244,1542,0x12c1d2e0c400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,311_Default/Mul-op7,Default/Mul-op7,kernel_graph0,{1333-1453-1540},1540,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1247,1545,0x12c21f00d600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,312_Default/Mul-op8,Default/Mul-op8,kernel_graph0,{1341-1453-1543},1543,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1250,1548,0x12c1dda0c600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,313_Default/Mul-op9,Default/Mul-op9,kernel_graph0,{1349-1453-1546},1546,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1253,1551,0x12c1e860c800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,314_Default/Mul-op10,Default/Mul-op10,kernel_graph0,{1357-1453-1549},1549,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1256,1554,0x12c1ec60ca00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,315_Default/Mul-op11,Default/Mul-op11,kernel_graph0,{1365-1453-1552},1552,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1259,1557,0x12c1f5c0ce00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,316_Default/Mul-op12,Default/Mul-op12,kernel_graph0,{1373-1453-1555},1555,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1262,1560,0x12c23aa0e600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,317_Default/Mul-op13,Default/Mul-op13,kernel_graph0,{1381-1453-1558},1558,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1265,1563,0x12c25020ea00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,318_Default/Mul-op14,Default/Mul-op14,kernel_graph0,{1389-1453-1561},1561,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1268,1566,0x12c25ee0ee00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,319_Default/Mul-op15,Default/Mul-op15,kernel_graph0,{1397-1453-1564},1564,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1271,1569,0x12c242a0ea00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,320_Default/Mul-op16,Default/Mul-op16,kernel_graph0,{1405-1453-1567},1567,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1274,1572,0x12c1afe1c800,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,321_Default/Mul-op17,Default/Mul-op17,kernel_graph0,{1413-1453-1570},1570,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1277,1575,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,322_Default/Mul-op18,Default/Mul-op18,kernel_graph0,{1421-1453-1573},1573,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1280,1578,0x12c1afe18600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,323_Default/Mul-op19,Default/Mul-op19,kernel_graph0,{1429-1453-1576},1576,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1283,1581,0x12c180011200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,324_Default/Mul-op20,Default/Mul-op20,kernel_graph0,{1437-1453-1579},1579,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1286,1584,0x12c184015600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,325_Default/Mul-op21,Default/Mul-op21,kernel_graph0,{1445-1453-1582},1582,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1289,1587,0x12c18000ca00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,326_Default/Mul-op22,Default/Mul-op22,kernel_graph0,{1453-1455-1585},1585,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1293,1298,0x12c18409d600,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,327_Default/clip_grad_norm-ClipGradNorm/Square-op0,Default/clip_grad_norm-ClipGradNorm/Square-op0,kernel_graph0,{1296},1296,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1295,1465,0x12c180010c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,328_Default/clip_grad_norm-ClipGradNorm/SumExt-op0,Default/clip_grad_norm-ClipGradNorm/SumExt-op0,kernel_graph0,{1299-1463},1463,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1296,1299,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,328_Default/clip_grad_norm-ClipGradNorm/SumExt-op0,Default/clip_grad_norm-ClipGradNorm/SumExt-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1301,1306,0x12c1f9c0d000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,330_Default/clip_grad_norm-ClipGradNorm/Square-op1,Default/clip_grad_norm-ClipGradNorm/Square-op1,kernel_graph0,{1304},1304,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1303,1466,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,331_Default/clip_grad_norm-ClipGradNorm/SumExt-op1,Default/clip_grad_norm-ClipGradNorm/SumExt-op1,kernel_graph0,{1307-1463},1463,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1304,1307,0x12c180004a00,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,331_Default/clip_grad_norm-ClipGradNorm/SumExt-op1,Default/clip_grad_norm-ClipGradNorm/SumExt-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1309,1314,0x12c1f9c0d000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,333_Default/clip_grad_norm-ClipGradNorm/Square-op2,Default/clip_grad_norm-ClipGradNorm/Square-op2,kernel_graph0,{1312},1312,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1311,1467,0x12c180004a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,334_Default/clip_grad_norm-ClipGradNorm/SumExt-op2,Default/clip_grad_norm-ClipGradNorm/SumExt-op2,kernel_graph0,{1315-1463},1463,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1312,1315,0x12c180004c00,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,334_Default/clip_grad_norm-ClipGradNorm/SumExt-op2,Default/clip_grad_norm-ClipGradNorm/SumExt-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1317,1322,0x12c1f9c0d000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,336_Default/clip_grad_norm-ClipGradNorm/Square-op3,Default/clip_grad_norm-ClipGradNorm/Square-op3,kernel_graph0,{1320},1320,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1319,1468,0x12c180004c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,337_Default/clip_grad_norm-ClipGradNorm/SumExt-op3,Default/clip_grad_norm-ClipGradNorm/SumExt-op3,kernel_graph0,{1323-1463},1463,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1320,1323,0x12c180004e00,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,337_Default/clip_grad_norm-ClipGradNorm/SumExt-op3,Default/clip_grad_norm-ClipGradNorm/SumExt-op3,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1325,1330,0x12c1f9c0d000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,339_Default/clip_grad_norm-ClipGradNorm/Square-op4,Default/clip_grad_norm-ClipGradNorm/Square-op4,kernel_graph0,{1328},1328,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1327,1469,0x12c180004e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,340_Default/clip_grad_norm-ClipGradNorm/SumExt-op4,Default/clip_grad_norm-ClipGradNorm/SumExt-op4,kernel_graph0,{1331-1463},1463,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1328,1331,0x12c180005000,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,340_Default/clip_grad_norm-ClipGradNorm/SumExt-op4,Default/clip_grad_norm-ClipGradNorm/SumExt-op4,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1333,1338,0x12c229c0d800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,342_Default/clip_grad_norm-ClipGradNorm/Square-op5,Default/clip_grad_norm-ClipGradNorm/Square-op5,kernel_graph0,{1336},1336,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1335,1470,0x12c180005000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,343_Default/clip_grad_norm-ClipGradNorm/SumExt-op5,Default/clip_grad_norm-ClipGradNorm/SumExt-op5,kernel_graph0,{1339-1463},1463,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1336,1339,0x12c180005200,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,343_Default/clip_grad_norm-ClipGradNorm/SumExt-op5,Default/clip_grad_norm-ClipGradNorm/SumExt-op5,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1341,1346,0x12c229c0d800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,345_Default/clip_grad_norm-ClipGradNorm/Square-op6,Default/clip_grad_norm-ClipGradNorm/Square-op6,kernel_graph0,{1344},1344,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1343,1471,0x12c180005200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,346_Default/clip_grad_norm-ClipGradNorm/SumExt-op6,Default/clip_grad_norm-ClipGradNorm/SumExt-op6,kernel_graph0,{1347-1463},1463,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1344,1347,0x12c180005400,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,346_Default/clip_grad_norm-ClipGradNorm/SumExt-op6,Default/clip_grad_norm-ClipGradNorm/SumExt-op6,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1349,1354,0x12c229c0d800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,348_Default/clip_grad_norm-ClipGradNorm/Square-op7,Default/clip_grad_norm-ClipGradNorm/Square-op7,kernel_graph0,{1352},1352,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1351,1472,0x12c180005400,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,349_Default/clip_grad_norm-ClipGradNorm/SumExt-op7,Default/clip_grad_norm-ClipGradNorm/SumExt-op7,kernel_graph0,{1355-1463},1463,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1352,1355,0x12c180005600,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,349_Default/clip_grad_norm-ClipGradNorm/SumExt-op7,Default/clip_grad_norm-ClipGradNorm/SumExt-op7,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1357,1362,0x12c1f9c0d000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,351_Default/clip_grad_norm-ClipGradNorm/Square-op8,Default/clip_grad_norm-ClipGradNorm/Square-op8,kernel_graph0,{1360},1360,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1359,1473,0x12c180005600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,352_Default/clip_grad_norm-ClipGradNorm/SumExt-op8,Default/clip_grad_norm-ClipGradNorm/SumExt-op8,kernel_graph0,{1363-1463},1463,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1360,1363,0x12c180005800,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,352_Default/clip_grad_norm-ClipGradNorm/SumExt-op8,Default/clip_grad_norm-ClipGradNorm/SumExt-op8,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1365,1370,0x12c1f9c0d000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,354_Default/clip_grad_norm-ClipGradNorm/Square-op9,Default/clip_grad_norm-ClipGradNorm/Square-op9,kernel_graph0,{1368},1368,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1367,1474,0x12c180005800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,355_Default/clip_grad_norm-ClipGradNorm/SumExt-op9,Default/clip_grad_norm-ClipGradNorm/SumExt-op9,kernel_graph0,{1371-1463},1463,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1368,1371,0x12c180005a00,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,355_Default/clip_grad_norm-ClipGradNorm/SumExt-op9,Default/clip_grad_norm-ClipGradNorm/SumExt-op9,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1373,1378,0x12c1f9c0d000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,357_Default/clip_grad_norm-ClipGradNorm/Square-op10,Default/clip_grad_norm-ClipGradNorm/Square-op10,kernel_graph0,{1376},1376,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1375,1475,0x12c180005a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,358_Default/clip_grad_norm-ClipGradNorm/SumExt-op10,Default/clip_grad_norm-ClipGradNorm/SumExt-op10,kernel_graph0,{1379-1463},1463,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1376,1379,0x12c180005c00,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,358_Default/clip_grad_norm-ClipGradNorm/SumExt-op10,Default/clip_grad_norm-ClipGradNorm/SumExt-op10,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1381,1386,0x12c1f9c0d000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,360_Default/clip_grad_norm-ClipGradNorm/Square-op11,Default/clip_grad_norm-ClipGradNorm/Square-op11,kernel_graph0,{1384},1384,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1383,1476,0x12c180005c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,361_Default/clip_grad_norm-ClipGradNorm/SumExt-op11,Default/clip_grad_norm-ClipGradNorm/SumExt-op11,kernel_graph0,{1387-1463},1463,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1384,1387,0x12c180005e00,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,361_Default/clip_grad_norm-ClipGradNorm/SumExt-op11,Default/clip_grad_norm-ClipGradNorm/SumExt-op11,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1389,1394,0x12c229c0d800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,363_Default/clip_grad_norm-ClipGradNorm/Square-op12,Default/clip_grad_norm-ClipGradNorm/Square-op12,kernel_graph0,{1392},1392,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1391,1477,0x12c180005e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,364_Default/clip_grad_norm-ClipGradNorm/SumExt-op12,Default/clip_grad_norm-ClipGradNorm/SumExt-op12,kernel_graph0,{1395-1463},1463,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1392,1395,0x12c180006000,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,364_Default/clip_grad_norm-ClipGradNorm/SumExt-op12,Default/clip_grad_norm-ClipGradNorm/SumExt-op12,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1397,1402,0x12c229c0d800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,366_Default/clip_grad_norm-ClipGradNorm/Square-op13,Default/clip_grad_norm-ClipGradNorm/Square-op13,kernel_graph0,{1400},1400,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1399,1478,0x12c180006000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,367_Default/clip_grad_norm-ClipGradNorm/SumExt-op13,Default/clip_grad_norm-ClipGradNorm/SumExt-op13,kernel_graph0,{1403-1463},1463,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1400,1403,0x12c180006200,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,367_Default/clip_grad_norm-ClipGradNorm/SumExt-op13,Default/clip_grad_norm-ClipGradNorm/SumExt-op13,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1405,1410,0x12c229c0d800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,369_Default/clip_grad_norm-ClipGradNorm/Square-op14,Default/clip_grad_norm-ClipGradNorm/Square-op14,kernel_graph0,{1408},1408,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1407,1479,0x12c180006200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,370_Default/clip_grad_norm-ClipGradNorm/SumExt-op14,Default/clip_grad_norm-ClipGradNorm/SumExt-op14,kernel_graph0,{1411-1463},1463,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1408,1411,0x12c180006400,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,370_Default/clip_grad_norm-ClipGradNorm/SumExt-op14,Default/clip_grad_norm-ClipGradNorm/SumExt-op14,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1413,1418,0x12c18409d600,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,372_Default/clip_grad_norm-ClipGradNorm/Square-op15,Default/clip_grad_norm-ClipGradNorm/Square-op15,kernel_graph0,{1416},1416,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1415,1480,0x12c180006400,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,373_Default/clip_grad_norm-ClipGradNorm/SumExt-op15,Default/clip_grad_norm-ClipGradNorm/SumExt-op15,kernel_graph0,{1419-1463},1463,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1416,1419,0x12c180006600,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,373_Default/clip_grad_norm-ClipGradNorm/SumExt-op15,Default/clip_grad_norm-ClipGradNorm/SumExt-op15,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1421,1426,0x12c180006600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,375_Default/clip_grad_norm-ClipGradNorm/Square-op16,Default/clip_grad_norm-ClipGradNorm/Square-op16,kernel_graph0,{1424},1424,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1423,1481,0x12c18000a800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,376_Default/clip_grad_norm-ClipGradNorm/SumExt-op16,Default/clip_grad_norm-ClipGradNorm/SumExt-op16,kernel_graph0,{1427-1463},1463,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1424,1427,0x12c18000aa00,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,376_Default/clip_grad_norm-ClipGradNorm/SumExt-op16,Default/clip_grad_norm-ClipGradNorm/SumExt-op16,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1429,1434,0x12c180006600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,378_Default/clip_grad_norm-ClipGradNorm/Square-op17,Default/clip_grad_norm-ClipGradNorm/Square-op17,kernel_graph0,{1432},1432,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1431,1482,0x12c18000aa00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,379_Default/clip_grad_norm-ClipGradNorm/SumExt-op17,Default/clip_grad_norm-ClipGradNorm/SumExt-op17,kernel_graph0,{1435-1463},1463,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1432,1435,0x12c18000ac00,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,379_Default/clip_grad_norm-ClipGradNorm/SumExt-op17,Default/clip_grad_norm-ClipGradNorm/SumExt-op17,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1437,1442,0x12c180006600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,381_Default/clip_grad_norm-ClipGradNorm/Square-op18,Default/clip_grad_norm-ClipGradNorm/Square-op18,kernel_graph0,{1440},1440,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1439,1483,0x12c18000ac00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,382_Default/clip_grad_norm-ClipGradNorm/SumExt-op18,Default/clip_grad_norm-ClipGradNorm/SumExt-op18,kernel_graph0,{1443-1463},1463,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1440,1443,0x12c18000ae00,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,382_Default/clip_grad_norm-ClipGradNorm/SumExt-op18,Default/clip_grad_norm-ClipGradNorm/SumExt-op18,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1445,1450,0x12c180006600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,384_Default/clip_grad_norm-ClipGradNorm/Square-op19,Default/clip_grad_norm-ClipGradNorm/Square-op19,kernel_graph0,{1448},1448,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1447,1484,0x12c18000ae00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,385_Default/clip_grad_norm-ClipGradNorm/SumExt-op19,Default/clip_grad_norm-ClipGradNorm/SumExt-op19,kernel_graph0,{1451-1463},1463,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1448,1451,0x12c18000b000,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,385_Default/clip_grad_norm-ClipGradNorm/SumExt-op19,Default/clip_grad_norm-ClipGradNorm/SumExt-op19,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1453,4813,0x12c18000b000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,387_Default/AllFinite-op0,Default/AllFinite-op0,kernel_graph0,{1489-1492-1494},1494,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1455,1460,0x12c180006600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,388_Default/clip_grad_norm-ClipGradNorm/Square-op20,Default/clip_grad_norm-ClipGradNorm/Square-op20,kernel_graph0,{1458},1458,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1457,1485,0x12c18000b200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,389_Default/clip_grad_norm-ClipGradNorm/SumExt-op20,Default/clip_grad_norm-ClipGradNorm/SumExt-op20,kernel_graph0,{1461-1463},1463,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1458,1461,0x12c18000b400,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,389_Default/clip_grad_norm-ClipGradNorm/SumExt-op20,Default/clip_grad_norm-ClipGradNorm/SumExt-op20,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1463,1488,0x12c18000b400,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,391_Default/clip_grad_norm-ClipGradNorm/AddN-op0,Default/clip_grad_norm-ClipGradNorm/AddN-op0,kernel_graph0,{1486},1486,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1486,4810,0x12c18000b200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,392_Default/clip_grad_norm-ClipGradNorm/Sqrt-op0,Default/clip_grad_norm-ClipGradNorm/Sqrt-op0,kernel_graph0,{1496-1498},1498,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1489,1519,0x12c180010c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,393_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Select-op1,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Select-op1,kernel_graph0,{1510-1515},1515,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1492,1636,0x12c18000b400,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,394_Default/LogicalNot-op0,Default/LogicalNot-op0,kernel_graph0,{1512},1512,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1494,1522,0x12c18000b600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,395_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/LogicalOr-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/LogicalOr-op0,kernel_graph0,{1520},1520,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1496,1500,0x12c18000b800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,396_Default/clip_grad_norm-ClipGradNorm/GreaterEqual-op0,Default/clip_grad_norm-ClipGradNorm/GreaterEqual-op0,kernel_graph0,{1498},1498,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1498,1503,0x12c18000ba00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,397_Default/clip_grad_norm-ClipGradNorm/Select-op0,Default/clip_grad_norm-ClipGradNorm/Select-op0,kernel_graph0,{1501},1501,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1501,1506,0x12c18000b800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,398_Default/learning_rate-CosineWithWarmUpLR/Add-op0,Default/learning_rate-CosineWithWarmUpLR/Add-op0,kernel_graph0,{1504},1504,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1504,1509,0x12c18000ba00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,399_Default/clip_grad_norm-ClipGradNorm/Div-op0,Default/clip_grad_norm-ClipGradNorm/Div-op0,kernel_graph0,{1507},1507,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1507,1588,0x12c18000b800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,400_Default/clip_grad_norm-ClipGradNorm/Minimum-op0,Default/clip_grad_norm-ClipGradNorm/Minimum-op0,kernel_graph0,{1525-1528-1531-1534-1537-1540-1543-1546-1549-1552-1555-1558-1561-1564-1567-1570-1573-1576-1579-1582-1585},1585,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1510,1518,0x12c18000ba00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,401_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Mul-op1,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Mul-op1,kernel_graph0,{1515},1515,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1512,1517,0x12c18000bc00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,402_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/LogicalAnd-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/LogicalAnd-op0,kernel_graph0,{1515},1515,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1515,1594,0x12c180011000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,403_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Select-op2,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Select-op2,kernel_graph0,{1592},1592,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1520,1590,0x12c180010c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,404_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Select-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Select-op0,kernel_graph0,{1588},1588,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1523,1592,0x12c18000b600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,405_Default/learning_rate-CosineWithWarmUpLR/Add-op1,Default/learning_rate-CosineWithWarmUpLR/Add-op1,kernel_graph0,{1590},1590,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1525,1795,0x12c184019800,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,406_Default/clip_grad_norm-ClipGradNorm/Mul-op0,Default/clip_grad_norm-ClipGradNorm/Mul-op0,kernel_graph0,{1791-1793},1793,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1528,1785,0x12c1a3419a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,407_Default/clip_grad_norm-ClipGradNorm/Mul-op1,Default/clip_grad_norm-ClipGradNorm/Mul-op1,kernel_graph0,{1781-1783},1783,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1531,1777,0x12c1f060cc00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,408_Default/clip_grad_norm-ClipGradNorm/Mul-op2,Default/clip_grad_norm-ClipGradNorm/Mul-op2,kernel_graph0,{1773-1775},1775,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1534,1769,0x12c180015400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,409_Default/clip_grad_norm-ClipGradNorm/Mul-op3,Default/clip_grad_norm-ClipGradNorm/Mul-op3,kernel_graph0,{1765-1767},1767,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1537,1761,0x12c23ea0e800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,410_Default/clip_grad_norm-ClipGradNorm/Mul-op4,Default/clip_grad_norm-ClipGradNorm/Mul-op4,kernel_graph0,{1757-1759},1759,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1540,1753,0x12c229c0d800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,411_Default/clip_grad_norm-ClipGradNorm/Mul-op5,Default/clip_grad_norm-ClipGradNorm/Mul-op5,kernel_graph0,{1749-1751},1751,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1543,1745,0x12c1cf21ca00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,412_Default/clip_grad_norm-ClipGradNorm/Mul-op6,Default/clip_grad_norm-ClipGradNorm/Mul-op6,kernel_graph0,{1741-1743},1743,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1546,1737,0x12c1f9c0d000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,413_Default/clip_grad_norm-ClipGradNorm/Mul-op7,Default/clip_grad_norm-ClipGradNorm/Mul-op7,kernel_graph0,{1733-1735},1735,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1549,1729,0x12c25ae0ec00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,414_Default/clip_grad_norm-ClipGradNorm/Mul-op8,Default/clip_grad_norm-ClipGradNorm/Mul-op8,kernel_graph0,{1725-1727},1727,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1552,1721,0x12c23480da00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,415_Default/clip_grad_norm-ClipGradNorm/Mul-op9,Default/clip_grad_norm-ClipGradNorm/Mul-op9,kernel_graph0,{1717-1719},1719,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1555,1713,0x12c1a7419c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,416_Default/clip_grad_norm-ClipGradNorm/Mul-op10,Default/clip_grad_norm-ClipGradNorm/Mul-op10,kernel_graph0,{1709-1711},1711,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1558,1705,0x12c1ab419e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,417_Default/clip_grad_norm-ClipGradNorm/Mul-op11,Default/clip_grad_norm-ClipGradNorm/Mul-op11,kernel_graph0,{1701-1703},1703,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1561,1697,0x12c1d9e1cc00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,418_Default/clip_grad_norm-ClipGradNorm/Mul-op12,Default/clip_grad_norm-ClipGradNorm/Mul-op12,kernel_graph0,{1693-1695},1695,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1564,1689,0x12c1e4a1ce00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,419_Default/clip_grad_norm-ClipGradNorm/Mul-op13,Default/clip_grad_norm-ClipGradNorm/Mul-op13,kernel_graph0,{1685-1687},1687,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1567,1681,0x12c24d60ec00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,420_Default/clip_grad_norm-ClipGradNorm/Mul-op14,Default/clip_grad_norm-ClipGradNorm/Mul-op14,kernel_graph0,{1677-1679},1679,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1570,1673,0x12c20480d200,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,421_Default/clip_grad_norm-ClipGradNorm/Mul-op15,Default/clip_grad_norm-ClipGradNorm/Mul-op15,kernel_graph0,{1669-1671},1671,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1573,1665,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,422_Default/clip_grad_norm-ClipGradNorm/Mul-op16,Default/clip_grad_norm-ClipGradNorm/Mul-op16,kernel_graph0,{1661-1663},1663,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1576,1657,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,423_Default/clip_grad_norm-ClipGradNorm/Mul-op17,Default/clip_grad_norm-ClipGradNorm/Mul-op17,kernel_graph0,{1653-1655},1655,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1579,1649,0x12c1ef61d000,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,424_Default/clip_grad_norm-ClipGradNorm/Mul-op18,Default/clip_grad_norm-ClipGradNorm/Mul-op18,kernel_graph0,{1645-1647},1647,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1582,1641,0x12c180011200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,425_Default/clip_grad_norm-ClipGradNorm/Mul-op19,Default/clip_grad_norm-ClipGradNorm/Mul-op19,kernel_graph0,{1637-1639},1639,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1585,1803,0x12c184015600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,426_Default/clip_grad_norm-ClipGradNorm/Mul-op20,Default/clip_grad_norm-ClipGradNorm/Mul-op20,kernel_graph0,{1799-1801},1801,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1595,1604,0x12c180011000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,430_Default/learning_rate-CosineWithWarmUpLR/Cast-op0,Default/learning_rate-CosineWithWarmUpLR/Cast-op0,kernel_graph0,{1597-1600-1602},1602,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1597,1599,0x12c180008a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,242,431_Default/learning_rate-CosineWithWarmUpLR/GreaterEqual-op0,Default/learning_rate-CosineWithWarmUpLR/GreaterEqual-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1600,1605,0x12c180008a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1599,435_Default/learning_rate-CosineWithWarmUpLR/Greater-op0,Default/learning_rate-CosineWithWarmUpLR/Greater-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1602,1608,0x12c180008c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1601,436_Default/TensorMove-op0,Default/TensorMove-op0,kernel_graph0,{1606},1606,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1606,1611,0x12c180008a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1605,440_Default/learning_rate-CosineWithWarmUpLR/Sub-op1,Default/learning_rate-CosineWithWarmUpLR/Sub-op1,kernel_graph0,{1609},1609,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1609,1614,0x12c180011000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1606,441_Default/learning_rate-CosineWithWarmUpLR/Div-op1,Default/learning_rate-CosineWithWarmUpLR/Div-op1,kernel_graph0,{1612},1612,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1612,1617,0x12c180008a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1606,442_Default/learning_rate-CosineWithWarmUpLR/Mul-op1,Default/learning_rate-CosineWithWarmUpLR/Mul-op1,kernel_graph0,{1615},1615,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1615,1620,0x12c180011000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1607,443_Default/learning_rate-CosineWithWarmUpLR/Cos-op0,Default/learning_rate-CosineWithWarmUpLR/Cos-op0,kernel_graph0,{1618},1618,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1618,1623,0x12c180008a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1607,444_Default/learning_rate-CosineWithWarmUpLR/Add-op3,Default/learning_rate-CosineWithWarmUpLR/Add-op3,kernel_graph0,{1621},1621,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1621,1626,0x12c180011000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1607,445_Default/learning_rate-CosineWithWarmUpLR/Mul-op2,Default/learning_rate-CosineWithWarmUpLR/Mul-op2,kernel_graph0,{1624},1624,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1624,1629,0x12c180008a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1607,446_Default/learning_rate-CosineWithWarmUpLR/Maximum-op0,Default/learning_rate-CosineWithWarmUpLR/Maximum-op0,kernel_graph0,{1627},1627,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1627,1632,0x12c180011000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1607,447_Default/learning_rate-CosineWithWarmUpLR/Mul-op3,Default/learning_rate-CosineWithWarmUpLR/Mul-op3,kernel_graph0,{1630},1630,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1630,1635,0x12c180008a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1607,448_Default/learning_rate-CosineWithWarmUpLR/Add-op4,Default/learning_rate-CosineWithWarmUpLR/Add-op4,kernel_graph0,{1633},1633,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1633,4811,0x12c180011000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1609,451_Default/Reshape-op0,Default/Reshape-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1637,2466,0x12c18000b400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1636,453_Default/optimizer-FP32StateAdamWeightDecay/Mul-op96,Default/optimizer-FP32StateAdamWeightDecay/Mul-op96,kernel_graph0,{2463},2463,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1639,1644,0x12c1ef621200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1637,454_Default/optimizer-FP32StateAdamWeightDecay/Square-op19,Default/optimizer-FP32StateAdamWeightDecay/Square-op19,kernel_graph0,{1642},1642,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1642,2462,0x12c180011200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1638,455_Default/optimizer-FP32StateAdamWeightDecay/Mul-op98,Default/optimizer-FP32StateAdamWeightDecay/Mul-op98,kernel_graph0,{2459},2459,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1645,2430,0x12c1ef621200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1638,456_Default/optimizer-FP32StateAdamWeightDecay/Mul-op91,Default/optimizer-FP32StateAdamWeightDecay/Mul-op91,kernel_graph0,{2427},2427,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1647,1652,0x12c1ef625400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1638,457_Default/optimizer-FP32StateAdamWeightDecay/Square-op18,Default/optimizer-FP32StateAdamWeightDecay/Square-op18,kernel_graph0,{1650},1650,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1650,2434,0x12c1ef61d000,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1638,458_Default/optimizer-FP32StateAdamWeightDecay/Mul-op93,Default/optimizer-FP32StateAdamWeightDecay/Mul-op93,kernel_graph0,{2431},2431,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1653,2398,0x12c1ef625400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1639,459_Default/optimizer-FP32StateAdamWeightDecay/Mul-op86,Default/optimizer-FP32StateAdamWeightDecay/Mul-op86,kernel_graph0,{2395},2395,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1655,1660,0x12c1ef629600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1640,460_Default/optimizer-FP32StateAdamWeightDecay/Square-op17,Default/optimizer-FP32StateAdamWeightDecay/Square-op17,kernel_graph0,{1658},1658,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1658,2402,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1640,461_Default/optimizer-FP32StateAdamWeightDecay/Mul-op88,Default/optimizer-FP32StateAdamWeightDecay/Mul-op88,kernel_graph0,{2399},2399,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1661,2366,0x12c1ef629600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1640,462_Default/optimizer-FP32StateAdamWeightDecay/Mul-op81,Default/optimizer-FP32StateAdamWeightDecay/Mul-op81,kernel_graph0,{2363},2363,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1663,1668,0x12c1ef62d800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1640,463_Default/optimizer-FP32StateAdamWeightDecay/Square-op16,Default/optimizer-FP32StateAdamWeightDecay/Square-op16,kernel_graph0,{1666},1666,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1666,2370,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1642,464_Default/optimizer-FP32StateAdamWeightDecay/Mul-op83,Default/optimizer-FP32StateAdamWeightDecay/Mul-op83,kernel_graph0,{2367},2367,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1669,2334,0x12c1af41a000,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1643,465_Default/optimizer-FP32StateAdamWeightDecay/Mul-op76,Default/optimizer-FP32StateAdamWeightDecay/Mul-op76,kernel_graph0,{2331},2331,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1671,1676,0x12c25ee0ee00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1643,466_Default/optimizer-FP32StateAdamWeightDecay/Square-op15,Default/optimizer-FP32StateAdamWeightDecay/Square-op15,kernel_graph0,{1674},1674,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1674,2338,0x12c20480d200,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1643,467_Default/optimizer-FP32StateAdamWeightDecay/Mul-op78,Default/optimizer-FP32StateAdamWeightDecay/Mul-op78,kernel_graph0,{2335},2335,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1677,2302,0x12c242a0ea00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1643,468_Default/optimizer-FP32StateAdamWeightDecay/Mul-op71,Default/optimizer-FP32StateAdamWeightDecay/Mul-op71,kernel_graph0,{2299},2299,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1679,1684,0x12c25ee0ee00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1645,469_Default/optimizer-FP32StateAdamWeightDecay/Square-op14,Default/optimizer-FP32StateAdamWeightDecay/Square-op14,kernel_graph0,{1682},1682,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1682,2306,0x12c24d60ec00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1646,470_Default/optimizer-FP32StateAdamWeightDecay/Mul-op73,Default/optimizer-FP32StateAdamWeightDecay/Mul-op73,kernel_graph0,{2303},2303,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1685,2270,0x12c25ee0ee00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1646,471_Default/optimizer-FP32StateAdamWeightDecay/Mul-op66,Default/optimizer-FP32StateAdamWeightDecay/Mul-op66,kernel_graph0,{2267},2267,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1687,1692,0x12c269a0f000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1646,472_Default/optimizer-FP32StateAdamWeightDecay/Square-op13,Default/optimizer-FP32StateAdamWeightDecay/Square-op13,kernel_graph0,{1690},1690,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1690,2274,0x12c1e4a1ce00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1646,473_Default/optimizer-FP32StateAdamWeightDecay/Mul-op68,Default/optimizer-FP32StateAdamWeightDecay/Mul-op68,kernel_graph0,{2271},2271,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1693,2238,0x12c269a0f000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1647,474_Default/optimizer-FP32StateAdamWeightDecay/Mul-op61,Default/optimizer-FP32StateAdamWeightDecay/Mul-op61,kernel_graph0,{2235},2235,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1695,1700,0x12c27460f200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1648,475_Default/optimizer-FP32StateAdamWeightDecay/Square-op12,Default/optimizer-FP32StateAdamWeightDecay/Square-op12,kernel_graph0,{1698},1698,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1698,2242,0x12c1d9e1cc00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1648,476_Default/optimizer-FP32StateAdamWeightDecay/Mul-op63,Default/optimizer-FP32StateAdamWeightDecay/Mul-op63,kernel_graph0,{2239},2239,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1701,2206,0x12c1f460ce00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1648,477_Default/optimizer-FP32StateAdamWeightDecay/Mul-op56,Default/optimizer-FP32StateAdamWeightDecay/Mul-op56,kernel_graph0,{2203},2203,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1703,1708,0x12c223c0d400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1648,478_Default/optimizer-FP32StateAdamWeightDecay/Square-op11,Default/optimizer-FP32StateAdamWeightDecay/Square-op11,kernel_graph0,{1706},1706,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1706,2210,0x12c1ab419e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1650,479_Default/optimizer-FP32StateAdamWeightDecay/Mul-op58,Default/optimizer-FP32StateAdamWeightDecay/Mul-op58,kernel_graph0,{2207},2207,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1709,2174,0x12c223c0d400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1651,480_Default/optimizer-FP32StateAdamWeightDecay/Mul-op51,Default/optimizer-FP32StateAdamWeightDecay/Mul-op51,kernel_graph0,{2171},2171,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1711,1716,0x12c23880dc00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1651,481_Default/optimizer-FP32StateAdamWeightDecay/Square-op10,Default/optimizer-FP32StateAdamWeightDecay/Square-op10,kernel_graph0,{1714},1714,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1714,2178,0x12c1a7419c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1651,482_Default/optimizer-FP32StateAdamWeightDecay/Mul-op53,Default/optimizer-FP32StateAdamWeightDecay/Mul-op53,kernel_graph0,{2175},2175,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1717,2142,0x12c23880dc00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1651,483_Default/optimizer-FP32StateAdamWeightDecay/Mul-op46,Default/optimizer-FP32StateAdamWeightDecay/Mul-op46,kernel_graph0,{2139},2139,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1719,1724,0x12c27460f200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1653,484_Default/optimizer-FP32StateAdamWeightDecay/Square-op9,Default/optimizer-FP32StateAdamWeightDecay/Square-op9,kernel_graph0,{1722},1722,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1722,2146,0x12c23480da00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1654,485_Default/optimizer-FP32StateAdamWeightDecay/Mul-op48,Default/optimizer-FP32StateAdamWeightDecay/Mul-op48,kernel_graph0,{2143},2143,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1725,2110,0x12c27460f200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1654,486_Default/optimizer-FP32StateAdamWeightDecay/Mul-op41,Default/optimizer-FP32StateAdamWeightDecay/Mul-op41,kernel_graph0,{2107},2107,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1727,1732,0x12c27860f400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1654,487_Default/optimizer-FP32StateAdamWeightDecay/Square-op8,Default/optimizer-FP32StateAdamWeightDecay/Square-op8,kernel_graph0,{1730},1730,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1730,2114,0x12c25820ee00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1654,488_Default/optimizer-FP32StateAdamWeightDecay/Mul-op43,Default/optimizer-FP32StateAdamWeightDecay/Mul-op43,kernel_graph0,{2111},2111,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1733,2078,0x12c27860f400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1655,489_Default/optimizer-FP32StateAdamWeightDecay/Mul-op36,Default/optimizer-FP32StateAdamWeightDecay/Mul-op36,kernel_graph0,{2075},2075,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1735,1740,0x12c28320f600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1656,490_Default/optimizer-FP32StateAdamWeightDecay/Square-op7,Default/optimizer-FP32StateAdamWeightDecay/Square-op7,kernel_graph0,{1738},1738,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1738,2082,0x12c1f860d000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1656,491_Default/optimizer-FP32StateAdamWeightDecay/Mul-op38,Default/optimizer-FP32StateAdamWeightDecay/Mul-op38,kernel_graph0,{2079},2079,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1741,2046,0x12c28320f600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1656,492_Default/optimizer-FP32StateAdamWeightDecay/Mul-op31,Default/optimizer-FP32StateAdamWeightDecay/Mul-op31,kernel_graph0,{2043},2043,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1743,1748,0x12c28de0f800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1656,493_Default/optimizer-FP32StateAdamWeightDecay/Square-op6,Default/optimizer-FP32StateAdamWeightDecay/Square-op6,kernel_graph0,{1746},1746,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1746,2050,0x12c1ce81a200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1658,494_Default/optimizer-FP32StateAdamWeightDecay/Mul-op33,Default/optimizer-FP32StateAdamWeightDecay/Mul-op33,kernel_graph0,{2047},2047,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1749,2014,0x12c28de0f800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1659,495_Default/optimizer-FP32StateAdamWeightDecay/Mul-op26,Default/optimizer-FP32StateAdamWeightDecay/Mul-op26,kernel_graph0,{2011},2011,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1751,1756,0x12c298a0fa00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1659,496_Default/optimizer-FP32StateAdamWeightDecay/Square-op5,Default/optimizer-FP32StateAdamWeightDecay/Square-op5,kernel_graph0,{1754},1754,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1754,2018,0x12c227c0d600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1659,497_Default/optimizer-FP32StateAdamWeightDecay/Mul-op28,Default/optimizer-FP32StateAdamWeightDecay/Mul-op28,kernel_graph0,{2015},2015,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1757,1982,0x12c298a0fa00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1659,498_Default/optimizer-FP32StateAdamWeightDecay/Mul-op21,Default/optimizer-FP32StateAdamWeightDecay/Mul-op21,kernel_graph0,{1979},1979,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1759,1764,0x12c29ca0fc00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1661,499_Default/optimizer-FP32StateAdamWeightDecay/Square-op4,Default/optimizer-FP32StateAdamWeightDecay/Square-op4,kernel_graph0,{1762},1762,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1762,1986,0x12c23c80de00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1662,500_Default/optimizer-FP32StateAdamWeightDecay/Mul-op23,Default/optimizer-FP32StateAdamWeightDecay/Mul-op23,kernel_graph0,{1983},1983,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1765,1950,0x12c29ca0fc00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1662,501_Default/optimizer-FP32StateAdamWeightDecay/Mul-op16,Default/optimizer-FP32StateAdamWeightDecay/Mul-op16,kernel_graph0,{1947},1947,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1767,1772,0x12c2a0a0fe00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1662,502_Default/optimizer-FP32StateAdamWeightDecay/Square-op3,Default/optimizer-FP32StateAdamWeightDecay/Square-op3,kernel_graph0,{1770},1770,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1770,1954,0x12c180015400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1662,503_Default/optimizer-FP32StateAdamWeightDecay/Mul-op18,Default/optimizer-FP32StateAdamWeightDecay/Mul-op18,kernel_graph0,{1951},1951,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1773,1918,0x12c2a0a0fe00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1663,504_Default/optimizer-FP32StateAdamWeightDecay/Mul-op11,Default/optimizer-FP32StateAdamWeightDecay/Mul-op11,kernel_graph0,{1915},1915,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1775,1780,0x12c2a4a10000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1664,505_Default/optimizer-FP32StateAdamWeightDecay/Square-op2,Default/optimizer-FP32StateAdamWeightDecay/Square-op2,kernel_graph0,{1778},1778,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1778,1922,0x12c1ef62d800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1664,506_Default/optimizer-FP32StateAdamWeightDecay/Mul-op13,Default/optimizer-FP32StateAdamWeightDecay/Mul-op13,kernel_graph0,{1919},1919,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1781,1886,0x12c2a4a10000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1664,507_Default/optimizer-FP32StateAdamWeightDecay/Mul-op6,Default/optimizer-FP32StateAdamWeightDecay/Mul-op6,kernel_graph0,{1883},1883,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1783,1788,0x12c2a8a10200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1664,508_Default/optimizer-FP32StateAdamWeightDecay/Square-op1,Default/optimizer-FP32StateAdamWeightDecay/Square-op1,kernel_graph0,{1786},1786,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1786,1890,0x12c1a3419a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1666,509_Default/optimizer-FP32StateAdamWeightDecay/Mul-op8,Default/optimizer-FP32StateAdamWeightDecay/Mul-op8,kernel_graph0,{1887},1887,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1789,1814,0x12c18000f600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1667,510_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Cast-op0,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Cast-op0,kernel_graph0,{1807-1810-1812},1812,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1791,1854,0x12c2a8a10200,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1667,511_Default/optimizer-FP32StateAdamWeightDecay/Mul-op1,Default/optimizer-FP32StateAdamWeightDecay/Mul-op1,kernel_graph0,{1851},1851,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1793,1798,0x12c2c7e10400,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1667,512_Default/optimizer-FP32StateAdamWeightDecay/Square-op0,Default/optimizer-FP32StateAdamWeightDecay/Square-op0,kernel_graph0,{1796},1796,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1796,1858,0x12c184019800,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1667,513_Default/optimizer-FP32StateAdamWeightDecay/Mul-op3,Default/optimizer-FP32StateAdamWeightDecay/Mul-op3,kernel_graph0,{1855},1855,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1799,2498,0x12c1d941a400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1669,514_Default/optimizer-FP32StateAdamWeightDecay/Mul-op101,Default/optimizer-FP32StateAdamWeightDecay/Mul-op101,kernel_graph0,{2495},2495,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1801,1806,0x12c1d941e600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1670,515_Default/optimizer-FP32StateAdamWeightDecay/Square-op20,Default/optimizer-FP32StateAdamWeightDecay/Square-op20,kernel_graph0,{1804},1804,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1804,2494,0x12c184015600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1670,516_Default/optimizer-FP32StateAdamWeightDecay/Mul-op103,Default/optimizer-FP32StateAdamWeightDecay/Mul-op103,kernel_graph0,{2491},2491,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1807,1809,0x12c18000f800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1670,518_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/GreaterEqual-op0,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/GreaterEqual-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1810,1815,0x12c18000f800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1809,522_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Greater-op0,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Greater-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1812,1818,0x12c18000fa00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1810,523_Default/TensorMove-op1,Default/TensorMove-op1,kernel_graph0,{1816},1816,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1816,1821,0x12c18000f600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1815,527_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Sub-op1,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Sub-op1,kernel_graph0,{1819},1819,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1819,1824,0x12c18000f800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1816,528_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Div-op1,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Div-op1,kernel_graph0,{1822},1822,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1822,1827,0x12c18000f600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1817,529_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Mul-op1,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Mul-op1,kernel_graph0,{1825},1825,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1825,1830,0x12c18000f800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1817,530_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Cos-op0,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Cos-op0,kernel_graph0,{1828},1828,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1828,1833,0x12c18000f600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1817,531_Default/learning_rate-CosineWithWarmUpLR/Add-op6,Default/learning_rate-CosineWithWarmUpLR/Add-op6,kernel_graph0,{1831},1831,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1831,1836,0x12c18000f800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1817,532_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Mul-op2,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Mul-op2,kernel_graph0,{1834},1834,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1834,1839,0x12c18000f600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1819,533_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Maximum-op0,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Maximum-op0,kernel_graph0,{1837},1837,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1837,1842,0x12c18000f800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1820,534_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Mul-op3,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Mul-op3,kernel_graph0,{1840},1840,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1840,1845,0x12c18000f600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1820,535_Default/learning_rate-CosineWithWarmUpLR/Add-op7,Default/learning_rate-CosineWithWarmUpLR/Add-op7,kernel_graph0,{1843},1843,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1843,2509,0x12c18000f800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1822,538_Default/optimizer-FP32StateAdamWeightDecay/Reshape-op0,Default/optimizer-FP32StateAdamWeightDecay/Reshape-op0,kernel_graph0,{1867-1899-1931-1963-1995-2027-2059-2091-2123-2155-2187-2219-2251-2283-2315-2347-2379-2411-2443-2475-2507},2507,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1847,1853,0x12c2c7e10400,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1823,540_Default/optimizer-FP32StateAdamWeightDecay/Mul-op0,Default/optimizer-FP32StateAdamWeightDecay/Mul-op0,kernel_graph0,{1851},1851,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1849,1857,0x12c2e7210600,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1823,541_Default/optimizer-FP32StateAdamWeightDecay/Mul-op2,Default/optimizer-FP32StateAdamWeightDecay/Mul-op2,kernel_graph0,{1855},1855,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1851,1876,0x12c306610800,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1823,542_Default/learning_rate-CosineWithWarmUpLR/Add-op8,Default/learning_rate-CosineWithWarmUpLR/Add-op8,kernel_graph0,{1864-1874},1874,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1855,1874,0x12c2a8a10200,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1825,543_Default/learning_rate-CosineWithWarmUpLR/Add-op9,Default/learning_rate-CosineWithWarmUpLR/Add-op9,kernel_graph0,{1859-1872},1872,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1859,1863,0x12c184019800,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1826,544_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op0,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op0,kernel_graph0,{1861},1861,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1861,1866,0x12c2c7e10400,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1826,545_Default/learning_rate-CosineWithWarmUpLR/Add-op10,Default/learning_rate-CosineWithWarmUpLR/Add-op10,kernel_graph0,{1864},1864,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1864,1869,0x12c184019800,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1826,546_Default/optimizer-FP32StateAdamWeightDecay/Div-op0,Default/optimizer-FP32StateAdamWeightDecay/Div-op0,kernel_graph0,{1867},1867,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1867,1872,0x12c2c7e10400,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1826,547_Default/optimizer-FP32StateAdamWeightDecay/Mul-op4,Default/optimizer-FP32StateAdamWeightDecay/Mul-op4,kernel_graph0,{1870},1870,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1870,1878,0x12c184019800,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1828,548_Default/optimizer-FP32StateAdamWeightDecay/Sub-op0,Default/optimizer-FP32StateAdamWeightDecay/Sub-op0,kernel_graph0,{1876},1876,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1879,1885,0x12c184019800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1830,552_Default/optimizer-FP32StateAdamWeightDecay/Mul-op5,Default/optimizer-FP32StateAdamWeightDecay/Mul-op5,kernel_graph0,{1883},1883,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1881,1889,0x12c188019a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1831,553_Default/optimizer-FP32StateAdamWeightDecay/Mul-op7,Default/optimizer-FP32StateAdamWeightDecay/Mul-op7,kernel_graph0,{1887},1887,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1883,1908,0x12c18c019c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1832,554_Default/learning_rate-CosineWithWarmUpLR/Add-op11,Default/learning_rate-CosineWithWarmUpLR/Add-op11,kernel_graph0,{1896-1906},1906,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1887,1906,0x12c184019800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1832,555_Default/learning_rate-CosineWithWarmUpLR/Add-op12,Default/learning_rate-CosineWithWarmUpLR/Add-op12,kernel_graph0,{1891-1904},1904,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1891,1895,0x12c188019a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1832,556_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op1,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op1,kernel_graph0,{1893},1893,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1893,1898,0x12c190019e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1832,557_Default/learning_rate-CosineWithWarmUpLR/Add-op13,Default/learning_rate-CosineWithWarmUpLR/Add-op13,kernel_graph0,{1896},1896,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1896,1901,0x12c188019a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1834,558_Default/optimizer-FP32StateAdamWeightDecay/Div-op1,Default/optimizer-FP32StateAdamWeightDecay/Div-op1,kernel_graph0,{1899},1899,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1899,1904,0x12c190019e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1835,559_Default/optimizer-FP32StateAdamWeightDecay/Mul-op9,Default/optimizer-FP32StateAdamWeightDecay/Mul-op9,kernel_graph0,{1902},1902,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1902,1910,0x12c188019a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1835,560_Default/optimizer-FP32StateAdamWeightDecay/Sub-op1,Default/optimizer-FP32StateAdamWeightDecay/Sub-op1,kernel_graph0,{1908},1908,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1911,1917,0x12c184019800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1838,564_Default/optimizer-FP32StateAdamWeightDecay/Mul-op10,Default/optimizer-FP32StateAdamWeightDecay/Mul-op10,kernel_graph0,{1915},1915,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1913,1921,0x12c188019a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1838,565_Default/optimizer-FP32StateAdamWeightDecay/Mul-op12,Default/optimizer-FP32StateAdamWeightDecay/Mul-op12,kernel_graph0,{1919},1919,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1915,1940,0x12c18c019c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1838,566_Default/learning_rate-CosineWithWarmUpLR/Add-op14,Default/learning_rate-CosineWithWarmUpLR/Add-op14,kernel_graph0,{1928-1938},1938,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1919,1938,0x12c184019800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1838,567_Default/learning_rate-CosineWithWarmUpLR/Add-op15,Default/learning_rate-CosineWithWarmUpLR/Add-op15,kernel_graph0,{1923-1936},1936,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1923,1927,0x12c188019a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1840,568_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op2,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op2,kernel_graph0,{1925},1925,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1925,1930,0x12c1ef62d800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1840,569_Default/learning_rate-CosineWithWarmUpLR/Add-op16,Default/learning_rate-CosineWithWarmUpLR/Add-op16,kernel_graph0,{1928},1928,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1928,1933,0x12c188019a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1841,570_Default/optimizer-FP32StateAdamWeightDecay/Div-op2,Default/optimizer-FP32StateAdamWeightDecay/Div-op2,kernel_graph0,{1931},1931,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1931,1936,0x12c1ef62d800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1841,571_Default/optimizer-FP32StateAdamWeightDecay/Mul-op14,Default/optimizer-FP32StateAdamWeightDecay/Mul-op14,kernel_graph0,{1934},1934,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1934,1942,0x12c188019a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1841,572_Default/optimizer-FP32StateAdamWeightDecay/Sub-op2,Default/optimizer-FP32StateAdamWeightDecay/Sub-op2,kernel_graph0,{1940},1940,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1943,1949,0x12c1ef62d800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1844,576_Default/optimizer-FP32StateAdamWeightDecay/Mul-op15,Default/optimizer-FP32StateAdamWeightDecay/Mul-op15,kernel_graph0,{1947},1947,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1945,1953,0x12c184019800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1845,577_Default/optimizer-FP32StateAdamWeightDecay/Mul-op17,Default/optimizer-FP32StateAdamWeightDecay/Mul-op17,kernel_graph0,{1951},1951,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1947,1972,0x12c188019a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,578_Default/learning_rate-CosineWithWarmUpLR/Add-op17,Default/learning_rate-CosineWithWarmUpLR/Add-op17,kernel_graph0,{1960-1970},1970,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1951,1970,0x12c1ef62d800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,579_Default/learning_rate-CosineWithWarmUpLR/Add-op18,Default/learning_rate-CosineWithWarmUpLR/Add-op18,kernel_graph0,{1955-1968},1968,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1955,1959,0x12c180015400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,580_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op3,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op3,kernel_graph0,{1957},1957,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1957,1962,0x12c184019800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,581_Default/learning_rate-CosineWithWarmUpLR/Add-op19,Default/learning_rate-CosineWithWarmUpLR/Add-op19,kernel_graph0,{1960},1960,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1960,1965,0x12c180015400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,582_Default/optimizer-FP32StateAdamWeightDecay/Div-op3,Default/optimizer-FP32StateAdamWeightDecay/Div-op3,kernel_graph0,{1963},1963,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1963,1968,0x12c184019800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,583_Default/optimizer-FP32StateAdamWeightDecay/Mul-op19,Default/optimizer-FP32StateAdamWeightDecay/Mul-op19,kernel_graph0,{1966},1966,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1966,1974,0x12c180015400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,584_Default/optimizer-FP32StateAdamWeightDecay/Sub-op3,Default/optimizer-FP32StateAdamWeightDecay/Sub-op3,kernel_graph0,{1972},1972,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1975,1981,0x12c180015400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,588_Default/optimizer-FP32StateAdamWeightDecay/Mul-op20,Default/optimizer-FP32StateAdamWeightDecay/Mul-op20,kernel_graph0,{1979},1979,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1977,1985,0x12c1ef62d800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,589_Default/optimizer-FP32StateAdamWeightDecay/Mul-op22,Default/optimizer-FP32StateAdamWeightDecay/Mul-op22,kernel_graph0,{1983},1983,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1979,2004,0x12c184019800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,590_Default/learning_rate-CosineWithWarmUpLR/Add-op20,Default/learning_rate-CosineWithWarmUpLR/Add-op20,kernel_graph0,{1992-2002},2002,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1983,2002,0x12c180015400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,591_Default/learning_rate-CosineWithWarmUpLR/Add-op21,Default/learning_rate-CosineWithWarmUpLR/Add-op21,kernel_graph0,{1987-2000},2000,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1987,1991,0x12c1ef62d800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,592_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op4,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op4,kernel_graph0,{1989},1989,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1989,1994,0x12c23c80de00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,593_Default/learning_rate-CosineWithWarmUpLR/Add-op22,Default/learning_rate-CosineWithWarmUpLR/Add-op22,kernel_graph0,{1992},1992,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1992,1997,0x12c1ef62d800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,594_Default/optimizer-FP32StateAdamWeightDecay/Div-op4,Default/optimizer-FP32StateAdamWeightDecay/Div-op4,kernel_graph0,{1995},1995,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1995,2000,0x12c23c80de00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,595_Default/optimizer-FP32StateAdamWeightDecay/Mul-op24,Default/optimizer-FP32StateAdamWeightDecay/Mul-op24,kernel_graph0,{1998},1998,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 1998,2006,0x12c1ef62d800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,596_Default/optimizer-FP32StateAdamWeightDecay/Sub-op4,Default/optimizer-FP32StateAdamWeightDecay/Sub-op4,kernel_graph0,{2004},2004,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2007,2013,0x12c184019800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,600_Default/optimizer-FP32StateAdamWeightDecay/Mul-op25,Default/optimizer-FP32StateAdamWeightDecay/Mul-op25,kernel_graph0,{2011},2011,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2009,2017,0x12c18ec19a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,601_Default/optimizer-FP32StateAdamWeightDecay/Mul-op27,Default/optimizer-FP32StateAdamWeightDecay/Mul-op27,kernel_graph0,{2015},2015,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2011,2036,0x12c199819c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,602_Default/learning_rate-CosineWithWarmUpLR/Add-op23,Default/learning_rate-CosineWithWarmUpLR/Add-op23,kernel_graph0,{2024-2034},2034,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2015,2034,0x12c184019800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,603_Default/learning_rate-CosineWithWarmUpLR/Add-op24,Default/learning_rate-CosineWithWarmUpLR/Add-op24,kernel_graph0,{2019-2032},2032,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2019,2023,0x12c18ec19a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,604_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op5,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op5,kernel_graph0,{2021},2021,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2021,2026,0x12c227c0d600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,605_Default/learning_rate-CosineWithWarmUpLR/Add-op25,Default/learning_rate-CosineWithWarmUpLR/Add-op25,kernel_graph0,{2024},2024,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2024,2029,0x12c18ec19a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,606_Default/optimizer-FP32StateAdamWeightDecay/Div-op5,Default/optimizer-FP32StateAdamWeightDecay/Div-op5,kernel_graph0,{2027},2027,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2027,2032,0x12c227c0d600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,607_Default/optimizer-FP32StateAdamWeightDecay/Mul-op29,Default/optimizer-FP32StateAdamWeightDecay/Mul-op29,kernel_graph0,{2030},2030,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2030,2038,0x12c18ec19a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,608_Default/optimizer-FP32StateAdamWeightDecay/Sub-op5,Default/optimizer-FP32StateAdamWeightDecay/Sub-op5,kernel_graph0,{2036},2036,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2039,2045,0x12c227c0d600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,612_Default/optimizer-FP32StateAdamWeightDecay/Mul-op30,Default/optimizer-FP32StateAdamWeightDecay/Mul-op30,kernel_graph0,{2043},2043,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2041,2049,0x12c184019800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,613_Default/optimizer-FP32StateAdamWeightDecay/Mul-op32,Default/optimizer-FP32StateAdamWeightDecay/Mul-op32,kernel_graph0,{2047},2047,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2043,2068,0x12c18ec19a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,614_Default/learning_rate-CosineWithWarmUpLR/Add-op26,Default/learning_rate-CosineWithWarmUpLR/Add-op26,kernel_graph0,{2056-2066},2066,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2047,2066,0x12c227c0d600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,615_Default/learning_rate-CosineWithWarmUpLR/Add-op27,Default/learning_rate-CosineWithWarmUpLR/Add-op27,kernel_graph0,{2051-2064},2064,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2051,2055,0x12c184019800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,616_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op6,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op6,kernel_graph0,{2053},2053,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2053,2058,0x12c1ce81a200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,617_Default/learning_rate-CosineWithWarmUpLR/Add-op28,Default/learning_rate-CosineWithWarmUpLR/Add-op28,kernel_graph0,{2056},2056,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2056,2061,0x12c184019800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,618_Default/optimizer-FP32StateAdamWeightDecay/Div-op6,Default/optimizer-FP32StateAdamWeightDecay/Div-op6,kernel_graph0,{2059},2059,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2059,2064,0x12c1ce81a200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,619_Default/optimizer-FP32StateAdamWeightDecay/Mul-op34,Default/optimizer-FP32StateAdamWeightDecay/Mul-op34,kernel_graph0,{2062},2062,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2062,2070,0x12c184019800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,620_Default/optimizer-FP32StateAdamWeightDecay/Sub-op6,Default/optimizer-FP32StateAdamWeightDecay/Sub-op6,kernel_graph0,{2068},2068,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2071,2077,0x12c1ce81a200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,624_Default/optimizer-FP32StateAdamWeightDecay/Mul-op35,Default/optimizer-FP32StateAdamWeightDecay/Mul-op35,kernel_graph0,{2075},2075,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2073,2081,0x12c227c0d600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,625_Default/optimizer-FP32StateAdamWeightDecay/Mul-op37,Default/optimizer-FP32StateAdamWeightDecay/Mul-op37,kernel_graph0,{2079},2079,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2075,2100,0x12c184019800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,626_Default/learning_rate-CosineWithWarmUpLR/Add-op29,Default/learning_rate-CosineWithWarmUpLR/Add-op29,kernel_graph0,{2088-2098},2098,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2079,2098,0x12c1ce81a200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,627_Default/learning_rate-CosineWithWarmUpLR/Add-op30,Default/learning_rate-CosineWithWarmUpLR/Add-op30,kernel_graph0,{2083-2096},2096,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2083,2087,0x12c1f860d000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,628_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op7,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op7,kernel_graph0,{2085},2085,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2085,2090,0x12c227c0d600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,629_Default/learning_rate-CosineWithWarmUpLR/Add-op31,Default/learning_rate-CosineWithWarmUpLR/Add-op31,kernel_graph0,{2088},2088,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2088,2093,0x12c1f860d000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,630_Default/optimizer-FP32StateAdamWeightDecay/Div-op7,Default/optimizer-FP32StateAdamWeightDecay/Div-op7,kernel_graph0,{2091},2091,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2091,2096,0x12c227c0d600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,631_Default/optimizer-FP32StateAdamWeightDecay/Mul-op39,Default/optimizer-FP32StateAdamWeightDecay/Mul-op39,kernel_graph0,{2094},2094,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2094,2102,0x12c1f860d000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,632_Default/optimizer-FP32StateAdamWeightDecay/Sub-op7,Default/optimizer-FP32StateAdamWeightDecay/Sub-op7,kernel_graph0,{2100},2100,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2103,2109,0x12c180015400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,636_Default/optimizer-FP32StateAdamWeightDecay/Mul-op40,Default/optimizer-FP32StateAdamWeightDecay/Mul-op40,kernel_graph0,{2107},2107,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2105,2113,0x12c1ef62d800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,637_Default/optimizer-FP32StateAdamWeightDecay/Mul-op42,Default/optimizer-FP32StateAdamWeightDecay/Mul-op42,kernel_graph0,{2111},2111,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2107,2132,0x12c23c80de00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,638_Default/learning_rate-CosineWithWarmUpLR/Add-op32,Default/learning_rate-CosineWithWarmUpLR/Add-op32,kernel_graph0,{2120-2130},2130,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2111,2130,0x12c180015400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,639_Default/learning_rate-CosineWithWarmUpLR/Add-op33,Default/learning_rate-CosineWithWarmUpLR/Add-op33,kernel_graph0,{2115-2128},2128,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2115,2119,0x12c1ef62d800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,640_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op8,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op8,kernel_graph0,{2117},2117,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2117,2122,0x12c25820ee00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,641_Default/learning_rate-CosineWithWarmUpLR/Add-op34,Default/learning_rate-CosineWithWarmUpLR/Add-op34,kernel_graph0,{2120},2120,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2120,2125,0x12c1ef62d800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,642_Default/optimizer-FP32StateAdamWeightDecay/Div-op8,Default/optimizer-FP32StateAdamWeightDecay/Div-op8,kernel_graph0,{2123},2123,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2123,2128,0x12c25820ee00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,643_Default/optimizer-FP32StateAdamWeightDecay/Mul-op44,Default/optimizer-FP32StateAdamWeightDecay/Mul-op44,kernel_graph0,{2126},2126,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2126,2134,0x12c1ef62d800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,644_Default/optimizer-FP32StateAdamWeightDecay/Sub-op8,Default/optimizer-FP32StateAdamWeightDecay/Sub-op8,kernel_graph0,{2132},2132,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2135,2141,0x12c180015400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,648_Default/optimizer-FP32StateAdamWeightDecay/Mul-op45,Default/optimizer-FP32StateAdamWeightDecay/Mul-op45,kernel_graph0,{2139},2139,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2137,2145,0x12c1ef62d800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,649_Default/optimizer-FP32StateAdamWeightDecay/Mul-op47,Default/optimizer-FP32StateAdamWeightDecay/Mul-op47,kernel_graph0,{2143},2143,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2139,2164,0x12c23c80de00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,650_Default/learning_rate-CosineWithWarmUpLR/Add-op35,Default/learning_rate-CosineWithWarmUpLR/Add-op35,kernel_graph0,{2152-2162},2162,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2143,2162,0x12c180015400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,651_Default/learning_rate-CosineWithWarmUpLR/Add-op36,Default/learning_rate-CosineWithWarmUpLR/Add-op36,kernel_graph0,{2147-2160},2160,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2147,2151,0x12c1ef62d800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,652_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op9,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op9,kernel_graph0,{2149},2149,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2149,2154,0x12c25820ee00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,653_Default/learning_rate-CosineWithWarmUpLR/Add-op37,Default/learning_rate-CosineWithWarmUpLR/Add-op37,kernel_graph0,{2152},2152,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2152,2157,0x12c1ef62d800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,654_Default/optimizer-FP32StateAdamWeightDecay/Div-op9,Default/optimizer-FP32StateAdamWeightDecay/Div-op9,kernel_graph0,{2155},2155,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2155,2160,0x12c25820ee00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,655_Default/optimizer-FP32StateAdamWeightDecay/Mul-op49,Default/optimizer-FP32StateAdamWeightDecay/Mul-op49,kernel_graph0,{2158},2158,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2158,2166,0x12c1ef62d800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,656_Default/optimizer-FP32StateAdamWeightDecay/Sub-op9,Default/optimizer-FP32StateAdamWeightDecay/Sub-op9,kernel_graph0,{2164},2164,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2167,2173,0x12c180015400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,660_Default/optimizer-FP32StateAdamWeightDecay/Mul-op50,Default/optimizer-FP32StateAdamWeightDecay/Mul-op50,kernel_graph0,{2171},2171,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2169,2177,0x12c1ef62d800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,661_Default/optimizer-FP32StateAdamWeightDecay/Mul-op52,Default/optimizer-FP32StateAdamWeightDecay/Mul-op52,kernel_graph0,{2175},2175,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2171,2196,0x12c25820ee00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,662_Default/learning_rate-CosineWithWarmUpLR/Add-op38,Default/learning_rate-CosineWithWarmUpLR/Add-op38,kernel_graph0,{2184-2194},2194,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2175,2194,0x12c180015400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,663_Default/learning_rate-CosineWithWarmUpLR/Add-op39,Default/learning_rate-CosineWithWarmUpLR/Add-op39,kernel_graph0,{2179-2192},2192,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2179,2183,0x12c1ef62d800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,664_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op10,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op10,kernel_graph0,{2181},2181,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2181,2186,0x12c1ce81a200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,665_Default/learning_rate-CosineWithWarmUpLR/Add-op40,Default/learning_rate-CosineWithWarmUpLR/Add-op40,kernel_graph0,{2184},2184,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2184,2189,0x12c1ef62d800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,666_Default/optimizer-FP32StateAdamWeightDecay/Div-op10,Default/optimizer-FP32StateAdamWeightDecay/Div-op10,kernel_graph0,{2187},2187,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2187,2192,0x12c1ce81a200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,667_Default/optimizer-FP32StateAdamWeightDecay/Mul-op54,Default/optimizer-FP32StateAdamWeightDecay/Mul-op54,kernel_graph0,{2190},2190,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2190,2198,0x12c1ef62d800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,668_Default/optimizer-FP32StateAdamWeightDecay/Sub-op10,Default/optimizer-FP32StateAdamWeightDecay/Sub-op10,kernel_graph0,{2196},2196,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2199,2205,0x12c180015400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,672_Default/optimizer-FP32StateAdamWeightDecay/Mul-op55,Default/optimizer-FP32StateAdamWeightDecay/Mul-op55,kernel_graph0,{2203},2203,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2201,2209,0x12c1ef62d800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,673_Default/optimizer-FP32StateAdamWeightDecay/Mul-op57,Default/optimizer-FP32StateAdamWeightDecay/Mul-op57,kernel_graph0,{2207},2207,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2203,2228,0x12c25820ee00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,674_Default/learning_rate-CosineWithWarmUpLR/Add-op41,Default/learning_rate-CosineWithWarmUpLR/Add-op41,kernel_graph0,{2216-2226},2226,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2207,2226,0x12c180015400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,675_Default/learning_rate-CosineWithWarmUpLR/Add-op42,Default/learning_rate-CosineWithWarmUpLR/Add-op42,kernel_graph0,{2211-2224},2224,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2211,2215,0x12c1ce81a200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,676_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op11,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op11,kernel_graph0,{2213},2213,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2213,2218,0x12c1d281a400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,677_Default/learning_rate-CosineWithWarmUpLR/Add-op43,Default/learning_rate-CosineWithWarmUpLR/Add-op43,kernel_graph0,{2216},2216,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2216,2221,0x12c1ce81a200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,678_Default/optimizer-FP32StateAdamWeightDecay/Div-op11,Default/optimizer-FP32StateAdamWeightDecay/Div-op11,kernel_graph0,{2219},2219,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2219,2224,0x12c1d281a400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,679_Default/optimizer-FP32StateAdamWeightDecay/Mul-op59,Default/optimizer-FP32StateAdamWeightDecay/Mul-op59,kernel_graph0,{2222},2222,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2222,2230,0x12c1ce81a200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,680_Default/optimizer-FP32StateAdamWeightDecay/Sub-op11,Default/optimizer-FP32StateAdamWeightDecay/Sub-op11,kernel_graph0,{2228},2228,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2231,2237,0x12c1ce81a200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,684_Default/optimizer-FP32StateAdamWeightDecay/Mul-op60,Default/optimizer-FP32StateAdamWeightDecay/Mul-op60,kernel_graph0,{2235},2235,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2233,2241,0x12c1ef62d800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,685_Default/optimizer-FP32StateAdamWeightDecay/Mul-op62,Default/optimizer-FP32StateAdamWeightDecay/Mul-op62,kernel_graph0,{2239},2239,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2235,2260,0x12c223c0d400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,686_Default/learning_rate-CosineWithWarmUpLR/Add-op44,Default/learning_rate-CosineWithWarmUpLR/Add-op44,kernel_graph0,{2248-2258},2258,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2239,2258,0x12c1ce81a200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,687_Default/learning_rate-CosineWithWarmUpLR/Add-op45,Default/learning_rate-CosineWithWarmUpLR/Add-op45,kernel_graph0,{2243-2256},2256,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2243,2247,0x12c1d941e600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,688_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op12,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op12,kernel_graph0,{2245},2245,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2245,2250,0x12c22e80d600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,689_Default/learning_rate-CosineWithWarmUpLR/Add-op46,Default/learning_rate-CosineWithWarmUpLR/Add-op46,kernel_graph0,{2248},2248,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2248,2253,0x12c1d941e600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,690_Default/optimizer-FP32StateAdamWeightDecay/Div-op12,Default/optimizer-FP32StateAdamWeightDecay/Div-op12,kernel_graph0,{2251},2251,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2251,2256,0x12c22e80d600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,691_Default/optimizer-FP32StateAdamWeightDecay/Mul-op64,Default/optimizer-FP32StateAdamWeightDecay/Mul-op64,kernel_graph0,{2254},2254,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2254,2262,0x12c1d941e600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,692_Default/optimizer-FP32StateAdamWeightDecay/Sub-op12,Default/optimizer-FP32StateAdamWeightDecay/Sub-op12,kernel_graph0,{2260},2260,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2263,2269,0x12c1ce81a200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,696_Default/optimizer-FP32StateAdamWeightDecay/Mul-op65,Default/optimizer-FP32StateAdamWeightDecay/Mul-op65,kernel_graph0,{2267},2267,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2265,2273,0x12c1d941e600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,697_Default/optimizer-FP32StateAdamWeightDecay/Mul-op67,Default/optimizer-FP32StateAdamWeightDecay/Mul-op67,kernel_graph0,{2271},2271,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2267,2292,0x12c1ef62d800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,698_Default/learning_rate-CosineWithWarmUpLR/Add-op47,Default/learning_rate-CosineWithWarmUpLR/Add-op47,kernel_graph0,{2280-2290},2290,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2271,2290,0x12c1ce81a200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,699_Default/learning_rate-CosineWithWarmUpLR/Add-op48,Default/learning_rate-CosineWithWarmUpLR/Add-op48,kernel_graph0,{2275-2288},2288,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2275,2279,0x12c1d941e600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,700_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op13,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op13,kernel_graph0,{2277},2277,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2277,2282,0x12c1e401e800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,701_Default/learning_rate-CosineWithWarmUpLR/Add-op49,Default/learning_rate-CosineWithWarmUpLR/Add-op49,kernel_graph0,{2280},2280,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2280,2285,0x12c1d941e600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,702_Default/optimizer-FP32StateAdamWeightDecay/Div-op13,Default/optimizer-FP32StateAdamWeightDecay/Div-op13,kernel_graph0,{2283},2283,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2283,2288,0x12c1e401e800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,703_Default/optimizer-FP32StateAdamWeightDecay/Mul-op69,Default/optimizer-FP32StateAdamWeightDecay/Mul-op69,kernel_graph0,{2286},2286,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2286,2294,0x12c1d941e600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,704_Default/optimizer-FP32StateAdamWeightDecay/Sub-op13,Default/optimizer-FP32StateAdamWeightDecay/Sub-op13,kernel_graph0,{2292},2292,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2295,2301,0x12c1ce81a200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,708_Default/optimizer-FP32StateAdamWeightDecay/Mul-op70,Default/optimizer-FP32StateAdamWeightDecay/Mul-op70,kernel_graph0,{2299},2299,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2297,2305,0x12c1ef62d800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,709_Default/optimizer-FP32StateAdamWeightDecay/Mul-op72,Default/optimizer-FP32StateAdamWeightDecay/Mul-op72,kernel_graph0,{2303},2303,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2299,2324,0x12c1d941e600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,710_Default/learning_rate-CosineWithWarmUpLR/Add-op50,Default/learning_rate-CosineWithWarmUpLR/Add-op50,kernel_graph0,{2312-2322},2322,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2303,2322,0x12c1ce81a200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,711_Default/learning_rate-CosineWithWarmUpLR/Add-op51,Default/learning_rate-CosineWithWarmUpLR/Add-op51,kernel_graph0,{2307-2320},2320,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2307,2311,0x12c1e401e800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,712_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op14,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op14,kernel_graph0,{2309},2309,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2309,2314,0x12c1ef62d800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,713_Default/learning_rate-CosineWithWarmUpLR/Add-op52,Default/learning_rate-CosineWithWarmUpLR/Add-op52,kernel_graph0,{2312},2312,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2312,2317,0x12c1e401e800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,714_Default/optimizer-FP32StateAdamWeightDecay/Div-op14,Default/optimizer-FP32StateAdamWeightDecay/Div-op14,kernel_graph0,{2315},2315,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2315,2320,0x12c1ef62d800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,715_Default/optimizer-FP32StateAdamWeightDecay/Mul-op74,Default/optimizer-FP32StateAdamWeightDecay/Mul-op74,kernel_graph0,{2318},2318,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2318,2326,0x12c1e401e800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,716_Default/optimizer-FP32StateAdamWeightDecay/Sub-op14,Default/optimizer-FP32StateAdamWeightDecay/Sub-op14,kernel_graph0,{2324},2324,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2327,2333,0x12c184019800,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,720_Default/optimizer-FP32StateAdamWeightDecay/Mul-op75,Default/optimizer-FP32StateAdamWeightDecay/Mul-op75,kernel_graph0,{2331},2331,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2329,2337,0x12c223c0d400,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,721_Default/optimizer-FP32StateAdamWeightDecay/Mul-op77,Default/optimizer-FP32StateAdamWeightDecay/Mul-op77,kernel_graph0,{2335},2335,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2331,2356,0x12c24300d600,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,722_Default/learning_rate-CosineWithWarmUpLR/Add-op53,Default/learning_rate-CosineWithWarmUpLR/Add-op53,kernel_graph0,{2344-2354},2354,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2335,2354,0x12c184019800,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,723_Default/learning_rate-CosineWithWarmUpLR/Add-op54,Default/learning_rate-CosineWithWarmUpLR/Add-op54,kernel_graph0,{2339-2352},2352,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2339,2343,0x12c1a3419a00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,724_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op15,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op15,kernel_graph0,{2341},2341,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2341,2346,0x12c1ef62d800,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,725_Default/learning_rate-CosineWithWarmUpLR/Add-op55,Default/learning_rate-CosineWithWarmUpLR/Add-op55,kernel_graph0,{2344},2344,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2344,2349,0x12c20ea2da00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,726_Default/optimizer-FP32StateAdamWeightDecay/Div-op15,Default/optimizer-FP32StateAdamWeightDecay/Div-op15,kernel_graph0,{2347},2347,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2347,2352,0x12c1ef62d800,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,727_Default/optimizer-FP32StateAdamWeightDecay/Mul-op79,Default/optimizer-FP32StateAdamWeightDecay/Mul-op79,kernel_graph0,{2350},2350,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2350,2358,0x12c20ea2da00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,728_Default/optimizer-FP32StateAdamWeightDecay/Sub-op15,Default/optimizer-FP32StateAdamWeightDecay/Sub-op15,kernel_graph0,{2356},2356,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2359,2365,0x12c180015400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,732_Default/optimizer-FP32StateAdamWeightDecay/Mul-op80,Default/optimizer-FP32StateAdamWeightDecay/Mul-op80,kernel_graph0,{2363},2363,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2361,2369,0x12c180019600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,733_Default/optimizer-FP32StateAdamWeightDecay/Mul-op82,Default/optimizer-FP32StateAdamWeightDecay/Mul-op82,kernel_graph0,{2367},2367,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2363,2388,0x12c18001d800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,734_Default/learning_rate-CosineWithWarmUpLR/Add-op56,Default/learning_rate-CosineWithWarmUpLR/Add-op56,kernel_graph0,{2376-2386},2386,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2367,2386,0x12c180015400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,735_Default/learning_rate-CosineWithWarmUpLR/Add-op57,Default/learning_rate-CosineWithWarmUpLR/Add-op57,kernel_graph0,{2371-2384},2384,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2371,2375,0x12c180019600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,736_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op16,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op16,kernel_graph0,{2373},2373,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2373,2378,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,737_Default/learning_rate-CosineWithWarmUpLR/Add-op58,Default/learning_rate-CosineWithWarmUpLR/Add-op58,kernel_graph0,{2376},2376,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2376,2381,0x12c180019600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,738_Default/optimizer-FP32StateAdamWeightDecay/Div-op16,Default/optimizer-FP32StateAdamWeightDecay/Div-op16,kernel_graph0,{2379},2379,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2379,2384,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,739_Default/optimizer-FP32StateAdamWeightDecay/Mul-op84,Default/optimizer-FP32StateAdamWeightDecay/Mul-op84,kernel_graph0,{2382},2382,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2382,2390,0x12c180019600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,740_Default/optimizer-FP32StateAdamWeightDecay/Sub-op16,Default/optimizer-FP32StateAdamWeightDecay/Sub-op16,kernel_graph0,{2388},2388,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2391,2397,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,744_Default/optimizer-FP32StateAdamWeightDecay/Mul-op85,Default/optimizer-FP32StateAdamWeightDecay/Mul-op85,kernel_graph0,{2395},2395,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2393,2401,0x12c180015400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,745_Default/optimizer-FP32StateAdamWeightDecay/Mul-op87,Default/optimizer-FP32StateAdamWeightDecay/Mul-op87,kernel_graph0,{2399},2399,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2395,2420,0x12c180019600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,746_Default/learning_rate-CosineWithWarmUpLR/Add-op59,Default/learning_rate-CosineWithWarmUpLR/Add-op59,kernel_graph0,{2408-2418},2418,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2399,2418,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,747_Default/learning_rate-CosineWithWarmUpLR/Add-op60,Default/learning_rate-CosineWithWarmUpLR/Add-op60,kernel_graph0,{2403-2416},2416,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2403,2407,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,748_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op17,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op17,kernel_graph0,{2405},2405,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2405,2410,0x12c180015400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,749_Default/learning_rate-CosineWithWarmUpLR/Add-op61,Default/learning_rate-CosineWithWarmUpLR/Add-op61,kernel_graph0,{2408},2408,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2408,2413,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,750_Default/optimizer-FP32StateAdamWeightDecay/Div-op17,Default/optimizer-FP32StateAdamWeightDecay/Div-op17,kernel_graph0,{2411},2411,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2411,2416,0x12c180015400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,751_Default/optimizer-FP32StateAdamWeightDecay/Mul-op89,Default/optimizer-FP32StateAdamWeightDecay/Mul-op89,kernel_graph0,{2414},2414,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2414,2422,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,752_Default/optimizer-FP32StateAdamWeightDecay/Sub-op17,Default/optimizer-FP32StateAdamWeightDecay/Sub-op17,kernel_graph0,{2420},2420,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2423,2429,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,756_Default/optimizer-FP32StateAdamWeightDecay/Mul-op90,Default/optimizer-FP32StateAdamWeightDecay/Mul-op90,kernel_graph0,{2427},2427,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2425,2433,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,757_Default/optimizer-FP32StateAdamWeightDecay/Mul-op92,Default/optimizer-FP32StateAdamWeightDecay/Mul-op92,kernel_graph0,{2431},2431,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2427,2452,0x12c180015400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,758_Default/learning_rate-CosineWithWarmUpLR/Add-op62,Default/learning_rate-CosineWithWarmUpLR/Add-op62,kernel_graph0,{2440-2450},2450,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2431,2450,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,759_Default/learning_rate-CosineWithWarmUpLR/Add-op63,Default/learning_rate-CosineWithWarmUpLR/Add-op63,kernel_graph0,{2435-2448},2448,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2435,2439,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,760_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op18,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op18,kernel_graph0,{2437},2437,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2437,2442,0x12c180019600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,761_Default/learning_rate-CosineWithWarmUpLR/Add-op64,Default/learning_rate-CosineWithWarmUpLR/Add-op64,kernel_graph0,{2440},2440,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2440,2445,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,762_Default/optimizer-FP32StateAdamWeightDecay/Div-op18,Default/optimizer-FP32StateAdamWeightDecay/Div-op18,kernel_graph0,{2443},2443,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2443,2448,0x12c180019600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,763_Default/optimizer-FP32StateAdamWeightDecay/Mul-op94,Default/optimizer-FP32StateAdamWeightDecay/Mul-op94,kernel_graph0,{2446},2446,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2446,2454,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,764_Default/optimizer-FP32StateAdamWeightDecay/Sub-op18,Default/optimizer-FP32StateAdamWeightDecay/Sub-op18,kernel_graph0,{2452},2452,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2455,2461,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,768_Default/optimizer-FP32StateAdamWeightDecay/Mul-op97,Default/optimizer-FP32StateAdamWeightDecay/Mul-op97,kernel_graph0,{2459},2459,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2457,2465,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,769_Default/optimizer-FP32StateAdamWeightDecay/Mul-op95,Default/optimizer-FP32StateAdamWeightDecay/Mul-op95,kernel_graph0,{2463},2463,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2459,2482,0x12c180015400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,770_Default/learning_rate-CosineWithWarmUpLR/Add-op66,Default/learning_rate-CosineWithWarmUpLR/Add-op66,kernel_graph0,{2467-2480},2480,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2463,2484,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,771_Default/learning_rate-CosineWithWarmUpLR/Add-op65,Default/learning_rate-CosineWithWarmUpLR/Add-op65,kernel_graph0,{2472-2482},2482,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2467,2471,0x12c180011200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,772_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op19,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op19,kernel_graph0,{2469},2469,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2469,2474,0x12c18000b400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,773_Default/learning_rate-CosineWithWarmUpLR/Add-op67,Default/learning_rate-CosineWithWarmUpLR/Add-op67,kernel_graph0,{2472},2472,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2472,2477,0x12c180011200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,774_Default/optimizer-FP32StateAdamWeightDecay/Div-op19,Default/optimizer-FP32StateAdamWeightDecay/Div-op19,kernel_graph0,{2475},2475,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2475,2480,0x12c18000b400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,775_Default/optimizer-FP32StateAdamWeightDecay/Mul-op99,Default/optimizer-FP32StateAdamWeightDecay/Mul-op99,kernel_graph0,{2478},2478,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2478,2486,0x12c180011200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,776_Default/optimizer-FP32StateAdamWeightDecay/Sub-op19,Default/optimizer-FP32StateAdamWeightDecay/Sub-op19,kernel_graph0,{2484},2484,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2487,2493,0x12c18000b400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,780_Default/optimizer-FP32StateAdamWeightDecay/Mul-op102,Default/optimizer-FP32StateAdamWeightDecay/Mul-op102,kernel_graph0,{2491},2491,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2489,2497,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,781_Default/optimizer-FP32StateAdamWeightDecay/Mul-op100,Default/optimizer-FP32StateAdamWeightDecay/Mul-op100,kernel_graph0,{2495},2495,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2491,2519,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,782_Default/learning_rate-CosineWithWarmUpLR/Add-op69,Default/learning_rate-CosineWithWarmUpLR/Add-op69,kernel_graph0,{2499-2517},2517,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2495,2517,0x12c18000b400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,783_Default/learning_rate-CosineWithWarmUpLR/Add-op68,Default/learning_rate-CosineWithWarmUpLR/Add-op68,kernel_graph0,{2504-2515},2515,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2499,2503,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,784_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op20,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op20,kernel_graph0,{2501},2501,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2501,2506,0x12c180011200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,785_Default/learning_rate-CosineWithWarmUpLR/Add-op70,Default/learning_rate-CosineWithWarmUpLR/Add-op70,kernel_graph0,{2504},2504,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2504,2510,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,786_Default/optimizer-FP32StateAdamWeightDecay/Div-op20,Default/optimizer-FP32StateAdamWeightDecay/Div-op20,kernel_graph0,{2507},2507,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2507,2513,0x12c180011200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,787_Default/optimizer-FP32StateAdamWeightDecay/Mul-op104,Default/optimizer-FP32StateAdamWeightDecay/Mul-op104,kernel_graph0,{2511},2511,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2511,2515,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,788_Default/optimizer-FP32StateAdamWeightDecay/Sub-op20,Default/optimizer-FP32StateAdamWeightDecay/Sub-op20,kernel_graph0,{2513},2513,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2520,4814,0x12c18000b400,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,1846,792_Default/TensorMove-op2,Default/TensorMove-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2524,4812,0x12c75d35ec00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/output_actor.cc,689,Other,2522,kernel_graph_0_OutputActor,UpdateOutputDeviceAddress,,{},,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 2525,2929,0x12c75d35ee00,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Kernel,2524,0_Default/network-LlamaForCausalLM/StridedSlice-op0,inputs0,0_Default/network-LlamaForCausalLM/StridedSlice-op0,{2526-2927},2927,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,1,0, 2526,3508,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2524,0_Default/network-LlamaForCausalLM/StridedSlice-op0,inputs0,0_Default/network-LlamaForCausalLM/StridedSlice-op0,{2528-2530-3505},3505,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2528,2534,0x12c180011200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2526,1_Default/network-LlamaForCausalLM/model-LlamaModel/tok_embeddings-LlamaEmbedding/Gather-op0,Default/network-LlamaForCausalLM/model-LlamaModel/tok_embeddings-LlamaEmbedding/Gather-op0,kernel_graph0,{2532},2532,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2530,2540,0x12c180008800,0,DefaultEnhancedAscendMemoryPool,8704,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2527,2_Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/NotEqual-op0,Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/NotEqual-op0,kernel_graph0,{2535-2538},2538,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2532,3486,0x12c188011400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2527,3_Default/network-LlamaForCausalLM/model-LlamaModel/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/Cast-op0,kernel_graph0,{2544-2673-3484},3484,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2535,2963,0x12c180011200,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2527,4_Default/network-LlamaForCausalLM/Cast-op1,Default/network-LlamaForCausalLM/Cast-op1,kernel_graph0,{2536-2555-2916-2960},2960,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2538,2569,0x12c18000b600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2527,6_Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Cast-op0,kernel_graph0,{2563-2567},2567,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2541,3412,0x12c180019400,0,DefaultEnhancedAscendMemoryPool,524800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2528,7_Default/network-LlamaForCausalLM/model-LlamaModel/freqs_mgr-FreqsMgr/StridedSlice-op0,Default/network-LlamaForCausalLM/model-LlamaModel/freqs_mgr-FreqsMgr/StridedSlice-op0,kernel_graph0,{2542-2606-2629-2759-2774-3147-3168-3389-3410},3410,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2544,2550,0x12c18c011600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2529,9_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op0,kernel_graph0,{2548},2548,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2546,2554,0x12c194011800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2529,10_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{2552},2552,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2547,3493,0x12c180099600,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2529,10_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{3489},3489,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2548,2551,0x12c1800a1800,0,DefaultEnhancedAscendMemoryPool,16777728,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2529,10_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2552,3464,0x12c1800a1800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2530,11_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op1,kernel_graph0,{2577-2582-2597-2620-3434-3448-3462},3462,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2555,2563,0x12c18000f800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2531,12_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/ReduceSum-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/ReduceSum-op0,kernel_graph0,{2561},2561,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2557,2559,0x12c18000fa00,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,2557,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2558,2560,0x12c18000fe00,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,2558,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2561,2974,0x12c18000fa00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2531,13_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Add-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Add-op0,kernel_graph0,{2904-2971},2971,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2565,2570,0x12c1840a1a00,0,DefaultEnhancedAscendMemoryPool,2097664,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2533,15_Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/ExpandDims-op0,Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/ExpandDims-op0,kernel_graph0,{2567},2567,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2567,2573,0x12c1842a1c00,0,DefaultEnhancedAscendMemoryPool,16777728,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2533,16_Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Mul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Mul-op0,kernel_graph0,{2571},2571,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2571,2577,0x12c1852a1e00,0,DefaultEnhancedAscendMemoryPool,16777728,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2535,17_Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Sub-op0,Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Sub-op0,kernel_graph0,{2573-2575},2575,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2575,3370,0x12c1840a1a00,0,DefaultEnhancedAscendMemoryPool,8389120,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2536,19_Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Cast-op1,kernel_graph0,{2658-2805-3123-3364},3364,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2579,3458,0x12c1848a1c00,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2537,21_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/Cast-op0,kernel_graph0,{2582-3455},3455,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2581,2589,0x12c18c011600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2538,22_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,kernel_graph0,{2584-2587},2587,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2582,2584,0x12c1868a1e00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2538,22_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2586,2593,0x12c190011800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2539,24_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,kernel_graph0,{2591},2591,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2587,2590,0x12c18000fc00,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2539,24_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2591,2610,0x12c194011a00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2541,25_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op2,kernel_graph0,{2606-2608},2608,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2594,3473,0x12c18c011600,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2542,26_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/Cast-op0,kernel_graph0,{2597-3470},3470,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2596,2604,0x12c18e011800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2542,27_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,kernel_graph0,{2599-2602},2602,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2597,2599,0x12c1868a1e00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2542,27_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2601,2613,0x12c19c011c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2542,29_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,kernel_graph0,{2611},2611,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2602,2605,0x12c18000fc00,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2542,29_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2606,2649,0x12c1a0011e00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2542,30_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op2,kernel_graph0,{2647},2647,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2608,2646,0x12c1a8012000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2543,31_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op1,kernel_graph0,{2644},2644,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2611,2633,0x12c18e011800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2544,32_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op0,kernel_graph0,{2629-2631},2631,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2614,3415,0x12c1868a1e00,0,DefaultEnhancedAscendMemoryPool,524800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2545,33_Default/network-LlamaForCausalLM/model-LlamaModel/freqs_mgr-FreqsMgr/StridedSlice-op1,Default/network-LlamaForCausalLM/model-LlamaModel/freqs_mgr-FreqsMgr/StridedSlice-op1,kernel_graph0,{2615-2634-2644-2764-2779-3149-3170-3391-3413},3413,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2617,3444,0x12c196011a00,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2548,35_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/Cast-op0,kernel_graph0,{2620-3441},3441,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2619,2627,0x12c198011c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2549,36_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,kernel_graph0,{2622-2625},2625,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2620,2622,0x12c186922000,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2549,36_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2624,3368,0x12c1b0012200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2550,38_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,kernel_graph0,{2658-3364},3364,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2625,2628,0x12c18000fc00,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2550,38_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2629,2639,0x12c198011c00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2552,39_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op0,kernel_graph0,{2637},2637,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2631,2636,0x12c1b4012400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2553,40_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op0,kernel_graph0,{2634},2634,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2634,2640,0x12c18e011800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2553,41_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op1,kernel_graph0,{2637},2637,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2637,2643,0x12c1b4012400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2555,42_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op0,kernel_graph0,{2641},2641,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2641,3366,0x12c18e011800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2556,43_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op1,kernel_graph0,{2658-3364},3364,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2644,2650,0x12c198011c00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2556,44_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op3,kernel_graph0,{2647},2647,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2647,2653,0x12c1a8012000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2556,45_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op1,kernel_graph0,{2651},2651,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2651,3367,0x12c198011c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2557,46_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op3,kernel_graph0,{2658-3364},3364,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2654,3371,0x12c186922000,0,DefaultEnhancedAscendMemoryPool,8389120,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2557,47_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{3364},3364,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2655,3372,0x12c187122200,0,DefaultEnhancedAscendMemoryPool,8389120,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2557,47_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{3364},3364,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2656,3373,0x12c18000fc00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2557,47_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{3364},3364,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2657,3374,0x12c19c011e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2557,47_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{2662-3364},3364,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2658,2660,0x12c1a0012000,0,DefaultEnhancedAscendMemoryPool,84936192,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2557,47_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2661,3344,0x12c1a0012000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2558,48_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,kernel_graph0,{2664-2669-3342},3342,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2662,2664,0x12c18000fe00,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2558,48_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2666,3352,0x12c192011a00,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2561,50_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/Cast-op0,kernel_graph0,{2669-3350},3350,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2668,2675,0x12c1a4012200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2561,51_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,kernel_graph0,{2671-2673},2673,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2669,2671,0x12c194011c00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2561,51_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2673,3324,0x12c1a8012400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2563,53_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/Add-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/Add-op0,kernel_graph0,{2676-2714-3322},3322,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2676,2682,0x12c1b4012400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2563,54_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op0,kernel_graph0,{2680},2680,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2678,2686,0x12c1bc012600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2564,55_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{2684},2684,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2679,3331,0x12c187922400,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2564,55_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{3327},3327,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2680,2683,0x12c194011c00,0,DefaultEnhancedAscendMemoryPool,16777728,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2564,55_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2684,3303,0x12c1a4012200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2565,56_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op1,kernel_graph0,{2686-2691-2698-3287-3301},3301,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2688,3297,0x12c1b4012400,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2567,58_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/Cast-op0,kernel_graph0,{2691-3294},3294,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2690,3279,0x12c1b9612600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2568,59_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,kernel_graph0,{2693-2704-3277},3277,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2691,2693,0x12c194011c00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2568,59_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2695,3312,0x12c1c4212800,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2571,61_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/Cast-op0,kernel_graph0,{2698-3309},3309,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2697,3285,0x12c1c9812a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2572,62_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,kernel_graph0,{2701-3282},3282,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2698,2700,0x12c194011c00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2572,62_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2701,3275,0x12c1d4412c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2572,63_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/SiLU-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/SiLU-op0,kernel_graph0,{2702-2704-3273},3273,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2704,3262,0x12c1df012e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2574,65_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/Mul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/Mul-op0,kernel_graph0,{2705-2710-3260},3260,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2707,3270,0x12c1e9c13000,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2576,67_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/Cast-op0,kernel_graph0,{2710-3268},3268,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2709,2716,0x12c1ef213200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2576,68_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,kernel_graph0,{2712-2714},2714,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2710,2712,0x12c194011c00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2576,68_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2714,3242,0x12c1f3213400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2578,70_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/Add-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/Add-op1,kernel_graph0,{2717-2820-3240},3240,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2717,2723,0x12c1f7213600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2579,71_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op0,kernel_graph0,{2721},2721,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2719,2727,0x12c1ff213800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2580,72_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{2725},2725,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2720,3249,0x12c18792a600,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2580,72_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{3245},3245,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2721,2724,0x12c194011c00,0,DefaultEnhancedAscendMemoryPool,16777728,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2580,72_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2725,3220,0x12c1ef213200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2580,73_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op1,kernel_graph0,{2727-2732-2747-2792-3190-3204-3218},3218,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2729,3214,0x12c1ac012600,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2583,75_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/Cast-op0,kernel_graph0,{2732-3211},3211,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2731,2739,0x12c1f7213600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2583,76_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,kernel_graph0,{2734-2737},2737,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2732,2734,0x12c1ae012800,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2583,76_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2736,2743,0x12c1fb213800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2585,78_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,kernel_graph0,{2741},2741,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2737,2740,0x12c18000fe00,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2585,78_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2741,2763,0x12c1ff213a00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2587,79_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op2,kernel_graph0,{2759-2761},2761,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2744,3229,0x12c1f7213600,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2588,80_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/Cast-op0,kernel_graph0,{2747-3226},3226,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2746,2754,0x12c1f9213800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2588,81_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,kernel_graph0,{2749-2752},2752,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2747,2749,0x12c1ae012800,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2588,81_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2751,2758,0x12c207213c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2592,83_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,kernel_graph0,{2756},2756,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2752,2755,0x12c18000fe00,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2592,83_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2756,2778,0x12c20b213e00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2592,84_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op0,kernel_graph0,{2774-2776},2776,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2759,2769,0x12c213214000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2594,85_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op2,kernel_graph0,{2767},2767,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2761,2766,0x12c21b214200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2595,86_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op1,kernel_graph0,{2764},2764,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2764,2770,0x12c1f9213800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2595,87_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op3,kernel_graph0,{2767},2767,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2767,2773,0x12c201213a00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2597,88_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op1,kernel_graph0,{2771},2771,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2771,3126,0x12c1f9213800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2598,89_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op3,kernel_graph0,{2805-3123},3123,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2774,2784,0x12c1fd213a00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2598,90_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op0,kernel_graph0,{2782},2782,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2776,2781,0x12c213214000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2599,91_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op0,kernel_graph0,{2779},2779,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2779,2785,0x12c205213c00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2600,92_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op1,kernel_graph0,{2782},2782,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2782,2788,0x12c20d213e00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2602,93_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op0,kernel_graph0,{2786},2786,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2786,3125,0x12c1fd213a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2602,94_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op1,kernel_graph0,{2805-3123},3123,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2789,3200,0x12c201213c00,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2603,95_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/Cast-op0,kernel_graph0,{2792-3197},3197,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2791,2799,0x12c203213e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2605,96_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,kernel_graph0,{2794-2797},2797,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2792,2794,0x12c1ae012800,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2605,96_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2796,3127,0x12c207214000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2607,98_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,kernel_graph0,{2805-3123},3123,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2797,2800,0x12c18000fe00,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2607,98_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2801,3129,0x12c1ae012800,0,DefaultEnhancedAscendMemoryPool,8389120,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2607,99_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{3123},3123,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2802,3130,0x12c1ae812a00,0,DefaultEnhancedAscendMemoryPool,8389120,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2607,99_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{3123},3123,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2803,3131,0x12c18000fe00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2607,99_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{3123},3123,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2804,3132,0x12c203213e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2607,99_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{2809-3123},3123,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2805,2807,0x12c20b214200,0,DefaultEnhancedAscendMemoryPool,84936192,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2607,99_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2808,3103,0x12c20b214200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2608,100_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,kernel_graph0,{2811-2816-3101},3101,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2809,2811,0x12c180010000,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2608,100_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2813,3111,0x12c20f214400,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2609,102_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/Cast-op0,kernel_graph0,{2816-3109},3109,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2815,2822,0x12c211214600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2611,103_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,kernel_graph0,{2818-2820},2820,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2816,2818,0x12c194011c00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2611,103_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2820,3078,0x12c215214800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2613,105_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/Add-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/Add-op0,kernel_graph0,{2823-2861-3076},3076,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2823,2829,0x12c219214a00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2614,106_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op0,kernel_graph0,{2827},2827,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2825,2833,0x12c221214c00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2615,107_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{2831},2831,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2826,3085,0x12c187932800,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2615,107_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{3081},3081,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2827,2830,0x12c194011c00,0,DefaultEnhancedAscendMemoryPool,16777728,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2615,107_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2831,3057,0x12c211214600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2615,108_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op1,kernel_graph0,{2833-2838-2845-3041-3055},3055,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2835,3051,0x12c219214a00,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2615,110_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/Cast-op0,kernel_graph0,{2838-3048},3048,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2837,3033,0x12c21e814c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2616,111_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,kernel_graph0,{2840-2851-3031},3031,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2838,2840,0x12c194011c00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2616,111_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2842,3066,0x12c229414e00,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2618,113_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/Cast-op0,kernel_graph0,{2845-3063},3063,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2844,3039,0x12c22ea15000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2618,114_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,kernel_graph0,{2848-3036},3036,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2845,2847,0x12c194011c00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2618,114_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2848,3029,0x12c239615200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2620,115_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/SiLU-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/SiLU-op0,kernel_graph0,{2849-2851-3027},3027,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2851,3016,0x12c244215400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2621,117_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/Mul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/Mul-op0,kernel_graph0,{2852-2857-3014},3014,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2854,3024,0x12c24ee15600,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2623,119_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/Cast-op0,kernel_graph0,{2857-3022},3022,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2856,2863,0x12c254415800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2625,120_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,kernel_graph0,{2859-2861},2861,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2857,2859,0x12c194011c00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2625,120_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2861,2995,0x12c258415a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2626,122_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/Add-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/Add-op1,kernel_graph0,{2864-2993},2993,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2864,2870,0x12c25c415c00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2629,123_Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Cast-op0,kernel_graph0,{2868},2868,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2866,2874,0x12c264415e00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2630,124_Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{2872},2872,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2867,3002,0x12c18793aa00,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2630,124_Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{2998},2998,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2868,2871,0x12c194011c00,0,DefaultEnhancedAscendMemoryPool,16777728,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2630,124_Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2872,2978,0x12c254415800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2630,125_Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Cast-op1,kernel_graph0,{2874-2879-2976},2976,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2876,2987,0x12c25c415c00,0,DefaultEnhancedAscendMemoryPool,262144512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2632,127_Default/network-LlamaForCausalLM/lm_head-Linear/Cast-op0,Default/network-LlamaForCausalLM/lm_head-Linear/Cast-op0,kernel_graph0,{2879-2984},2984,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2878,2884,0x12c26be15e00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2632,128_Default/network-LlamaForCausalLM/lm_head-Linear/MatMul-op0,Default/network-LlamaForCausalLM/lm_head-Linear/MatMul-op0,kernel_graph0,{2882},2882,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2879,2881,0x12c194011c00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2632,128_Default/network-LlamaForCausalLM/lm_head-Linear/MatMul-op0,Default/network-LlamaForCausalLM/lm_head-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2882,2891,0x12c28b216000,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2634,129_Default/network-LlamaForCausalLM/Cast-op0,Default/network-LlamaForCausalLM/Cast-op0,kernel_graph0,{2886-2889},2889,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2885,2892,0x12c187942c00,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2635,130_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/ReduceMax-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/ReduceMax-op0,kernel_graph0,{2889},2889,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2886,2888,0x12c180010000,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2635,130_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/ReduceMax-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/ReduceMax-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2889,2925,0x12c2c9a16200,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2635,131_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Sub-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Sub-op0,kernel_graph0,{2895-2923},2923,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2893,2906,0x12c180010000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2637,132_Default/Mul-op0,Default/Mul-op0,kernel_graph0,{2904},2904,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2895,2903,0x12c26be15e00,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2638,133_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Exp-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Exp-op0,kernel_graph0,{2897},2897,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2897,2922,0x12c187942c00,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2638,134_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/ReduceSum-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/ReduceSum-op0,kernel_graph0,{2920},2920,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2899,2901,0x12c180010200,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,2899,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2900,2902,0x12c180010600,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,2900,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2904,2910,0x12c180010200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2641,135_Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_RealDiv/RealDiv-op0,Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_RealDiv/RealDiv-op0,kernel_graph0,{2908},2908,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2907,2915,0x12c180010000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2642,136_Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_RealDiv/SumExt-op0,Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_RealDiv/SumExt-op0,kernel_graph0,{2911-2913},2913,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2908,2911,0x12c180010400,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2642,136_Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_RealDiv/SumExt-op0,Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_RealDiv/SumExt-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2913,2918,0x12c18794ae00,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2644,138_Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_ReduceSum/Tile-op0,Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_ReduceSum/Tile-op0,kernel_graph0,{2916},2916,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2916,2942,0x12c187953000,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2644,139_Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_Mul/Mul-op0,Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_Mul/Mul-op0,kernel_graph0,{2918-2939},2939,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2920,2926,0x12c18794ae00,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2645,141_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Log-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Log-op0,kernel_graph0,{2923},2923,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2923,2948,0x12c26be15e00,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2647,142_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Sub-op1,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Sub-op1,kernel_graph0,{2934-2946},2946,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2927,2933,0x12c187942c00,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2648,143_Default/network-LlamaForCausalLM/StridedSlice-op1,Default/network-LlamaForCausalLM/StridedSlice-op1,kernel_graph0,{2929-2931},2931,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2931,2949,0x12c2aa616000,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2651,145_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/OneHot-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/OneHot-op0,kernel_graph0,{2936-2946},2946,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2934,2938,0x12c2e8e16200,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2652,146_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Exp-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Exp-op0,kernel_graph0,{2936},2936,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2936,2941,0x12c327616400,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2652,147_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Sub-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Sub-op0,kernel_graph0,{2939},2939,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2939,2945,0x12c2e8e16200,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2656,148_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Mul-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Mul-op0,kernel_graph0,{2943},2943,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2943,2986,0x12c327616400,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2658,149_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op0,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op0,kernel_graph0,{2976-2984},2984,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2946,2952,0x12c2e8e16200,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2659,150_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Mul-op1,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Mul-op1,kernel_graph0,{2950},2950,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2950,2959,0x12c26be15e00,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2659,151_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Neg-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Neg-op0,kernel_graph0,{2953},2953,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2953,2962,0x12c187942c00,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2659,152_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/ReduceSum-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/ReduceSum-op0,kernel_graph0,{2960},2960,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2955,2957,0x12c180010000,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,2955,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2956,2958,0x12c180010400,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,2956,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2960,2970,0x12c18794ae00,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2662,153_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Mul-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Mul-op0,kernel_graph0,{2964},2964,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2964,2973,0x12c180010000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2663,154_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/ReduceSum-op1,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/ReduceSum-op1,kernel_graph0,{2971},2971,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2966,2968,0x12c180010200,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,2966,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2967,2969,0x12c180010600,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,2967,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2971,4807,0x12c180010200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2663,155_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/RealDiv-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/RealDiv-op0,kernel_graph0,{4805},4805,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2975,2982,0x12c346a16600,0,DefaultEnhancedAscendMemoryPool,262144512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2664,156_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op29,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op29,kernel_graph0,{2980},2980,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2976,2979,0x12c194011c00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2664,156_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op29,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op29,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2980,3561,0x12c26be15e00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2665,157_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op29,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op29,kernel_graph0,{3559},3559,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2983,2992,0x12c254415800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2666,158_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op0,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op0,kernel_graph0,{2988-2990},2990,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2984,2988,0x12c346a16600,0,DefaultEnhancedAscendMemoryPool,155190272,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2666,158_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op0,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2990,3000,0x12c25c415c00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2667,160_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op1,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op1,kernel_graph0,{2998},2998,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2993,3001,0x12c28b216000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2669,161_Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Cast-op2,kernel_graph0,{2998},2998,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2996,3006,0x12c254415800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2670,162_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op0,kernel_graph0,{3004},3004,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2997,3576,0x12c18000b600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2670,162_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op0,kernel_graph0,{3574},3574,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 2998,3003,0x12c194011c00,0,DefaultEnhancedAscendMemoryPool,16778752,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2670,162_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3004,3093,0x12c25c415a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2670,163_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op0,kernel_graph0,{3008-3014-3022-3090},3090,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3007,3012,0x12c180010000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2671,164_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Mul-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Mul-op0,kernel_graph0,{3010},3010,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3010,3776,0x12c18000f800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2673,166_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Maximum-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Maximum-op0,kernel_graph0,{3774},3774,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3013,3020,0x12c254415800,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2674,167_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op27,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op27,kernel_graph0,{3018},3018,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3014,3017,0x12c194011c00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2674,167_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op27,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op27,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3018,3555,0x12c244215400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2674,168_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op27,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op27,kernel_graph0,{3553},3553,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3021,3034,0x12c260415c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2676,169_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op1,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op1,kernel_graph0,{3025-3027-3031},3031,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3022,3025,0x12c194011c00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2676,169_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op1,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3027,3050,0x12c24ee15600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2677,171_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op0,kernel_graph0,{3029-3041-3048},3048,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3031,3038,0x12c239615200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2680,173_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op1,kernel_graph0,{3034-3036},3036,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3036,3065,0x12c21e814c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2681,175_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/Grad_SiLU/SiLUGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/Grad_SiLU/SiLUGrad-op0,kernel_graph0,{3055-3063},3063,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3040,3046,0x12c260415c00,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2684,176_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op28,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op28,kernel_graph0,{3044},3044,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3041,3043,0x12c194011c00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2684,176_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op28,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op28,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3044,3558,0x12c22ea15000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2685,177_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op28,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op28,kernel_graph0,{3556},3556,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3047,3071,0x12c239615200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2685,178_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op2,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op2,kernel_graph0,{3052-3069},3069,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3048,3052,0x12c194011c00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2685,178_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op2,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3054,3061,0x12c219214a00,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2687,180_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op26,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op26,kernel_graph0,{3059},3059,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3055,3058,0x12c194011c00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2687,180_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op26,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op26,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3059,3552,0x12c260415c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2688,181_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op26,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op26,kernel_graph0,{3550},3550,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3062,3072,0x12c211214600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2689,182_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op3,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op3,kernel_graph0,{3067-3069},3069,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3063,3067,0x12c194011c00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2689,182_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op3,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op3,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3069,3075,0x12c23d615400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2691,184_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/AddN-op0,kernel_graph0,{3073},3073,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3073,3083,0x12c24ee15600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2692,185_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op2,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op2,kernel_graph0,{3081},3081,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3076,3084,0x12c239615200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2692,186_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op2,kernel_graph0,{3081},3081,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3079,3089,0x12c211214600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2693,187_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op1,kernel_graph0,{3087},3087,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3080,3570,0x12c180011200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2693,187_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op1,kernel_graph0,{3568},3568,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3081,3086,0x12c194011c00,0,DefaultEnhancedAscendMemoryPool,16778752,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2693,187_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3087,3092,0x12c239615200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2694,188_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op1,kernel_graph0,{3090},3090,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3090,3257,0x12c23d615400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2695,189_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/AddN-op0,kernel_graph0,{3095-3101-3109-3254},3254,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3094,3099,0x12c18000fa00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2696,190_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Sub-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Sub-op0,kernel_graph0,{3097},3097,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3097,3799,0x12c180010000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2698,192_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/LessEqual-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/LessEqual-op0,kernel_graph0,{3779-3797},3797,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3100,3107,0x12c241615600,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2699,193_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op25,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op25,kernel_graph0,{3105},3105,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3101,3104,0x12c194011c00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2699,193_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op25,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op25,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3105,3549,0x12c20b214200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2700,194_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op25,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op25,kernel_graph0,{3547},3547,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3108,3117,0x12c239615200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2701,195_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op4,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op4,kernel_graph0,{3112-3115},3115,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3109,3112,0x12c194011c00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2701,195_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op4,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op4,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3114,3128,0x12c24ee15600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2702,197_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op0,kernel_graph0,{3123},3123,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3115,3118,0x12c180010400,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2702,197_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3119,3140,0x12c239615200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2702,198_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,kernel_graph0,{3138},3138,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3120,3137,0x12c252e15800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2702,198_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,kernel_graph0,{3135},3135,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3121,3144,0x12c256e15a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2702,198_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,kernel_graph0,{3142},3142,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3122,3133,0x12c18000fa00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2702,198_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3123,3134,0x12c20f214400,0,DefaultEnhancedAscendMemoryPool,515969024,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2702,198_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3135,3151,0x12c1f9213800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2703,199_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op5,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op5,kernel_graph0,{3147-3149},3149,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3138,3172,0x12c203213e00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2704,200_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op3,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op3,kernel_graph0,{3168-3170},3170,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3141,3199,0x12c239615200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2705,201_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op3,kernel_graph0,{3145-3190-3197},3197,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3142,3145,0x12c180010400,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2705,201_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op3,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3147,3157,0x12c24ee15600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2705,203_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op4,kernel_graph0,{3155},3155,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3149,3154,0x12c256e15800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2706,204_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op5,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op5,kernel_graph0,{3152},3152,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3152,3158,0x12c1f9213800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2707,205_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op1,kernel_graph0,{3155},3155,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3155,3161,0x12c256e15800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2708,206_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op1,kernel_graph0,{3159},3159,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3159,3165,0x12c24ee15600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2709,207_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op6,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op6,kernel_graph0,{3163},3163,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3162,3213,0x12c1f9213800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2710,208_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op2,kernel_graph0,{3166-3204-3211},3211,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3163,3166,0x12c180010400,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2710,208_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3168,3178,0x12c24ee15600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2712,210_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op2,kernel_graph0,{3176},3176,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3170,3175,0x12c256e15800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2712,211_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op3,kernel_graph0,{3173},3173,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3173,3179,0x12c203213e00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2713,212_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op0,kernel_graph0,{3176},3176,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3176,3182,0x12c256e15800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2714,213_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op0,kernel_graph0,{3180},3180,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3180,3186,0x12c1fd213a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2715,214_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op4,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op4,kernel_graph0,{3184},3184,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3183,3228,0x12c203213e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2716,215_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op1,kernel_graph0,{3187-3218-3226},3226,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3184,3187,0x12c180010400,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2716,215_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3189,3195,0x12c241615600,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2718,217_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op24,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op24,kernel_graph0,{3193},3193,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3190,3192,0x12c1ae012800,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2718,217_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op24,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op24,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3193,3546,0x12c1fd213a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2719,218_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op24,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op24,kernel_graph0,{3544},3544,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3196,3236,0x12c207214000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2721,219_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op7,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op7,kernel_graph0,{3201-3232},3232,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3197,3201,0x12c1ae012800,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2721,219_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op7,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op7,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3203,3209,0x12c201213c00,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2722,221_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op23,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op23,kernel_graph0,{3207},3207,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3204,3206,0x12c1ae012800,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2722,221_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op23,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op23,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3207,3543,0x12c239615200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2725,222_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op23,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op23,kernel_graph0,{3541},3541,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3210,3235,0x12c24ee15600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2726,223_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op6,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op6,kernel_graph0,{3215-3232},3232,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3211,3215,0x12c1ae012800,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2726,223_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op6,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op6,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3217,3224,0x12c201213c00,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2727,225_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op22,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op22,kernel_graph0,{3222},3222,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3218,3221,0x12c194011c00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2727,225_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op22,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op22,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3222,3540,0x12c1ef213200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2728,226_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op22,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op22,kernel_graph0,{3538},3538,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3225,3234,0x12c1f9213800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2729,227_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op5,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op5,kernel_graph0,{3230-3232},3232,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3226,3230,0x12c194011c00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2729,227_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op5,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op5,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3232,3239,0x12c201213c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2731,229_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/AddN-op0,kernel_graph0,{3237},3237,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3237,3247,0x12c24ee15600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2732,230_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op7,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op7,kernel_graph0,{3245},3245,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3240,3248,0x12c256e15800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2733,231_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op2,kernel_graph0,{3245},3245,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3243,3253,0x12c1f3213400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2733,232_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op2,kernel_graph0,{3251},3251,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3244,3573,0x12c187932800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2733,232_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op2,kernel_graph0,{3571},3571,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3245,3250,0x12c25ee15a00,0,DefaultEnhancedAscendMemoryPool,16778752,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2733,232_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3251,3256,0x12c201213c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2734,233_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op2,kernel_graph0,{3254},3254,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3254,3339,0x12c205213e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2735,234_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/AddN-op0,kernel_graph0,{3257-3260-3268-3336},3336,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3259,3266,0x12c23d615400,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2738,236_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op20,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op20,kernel_graph0,{3264},3264,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3260,3263,0x12c242c15600,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2738,236_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op20,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op20,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3264,3534,0x12c1df012e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2738,237_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op20,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op20,kernel_graph0,{3532},3532,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3267,3280,0x12c24ee15600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2741,238_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op8,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op8,kernel_graph0,{3271-3273-3277},3277,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3268,3271,0x12c194011c00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2741,238_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op8,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op8,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3273,3296,0x12c20f214400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2742,240_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op6,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op6,kernel_graph0,{3275-3287-3294},3294,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3277,3284,0x12c1d4412c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2745,242_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op7,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op7,kernel_graph0,{3280-3282},3282,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3282,3311,0x12c1b9612600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2747,244_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/Grad_SiLU/SiLUGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/Grad_SiLU/SiLUGrad-op1,kernel_graph0,{3301-3309},3309,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3286,3292,0x12c1e9c13000,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2748,245_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op21,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op21,kernel_graph0,{3290},3290,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3287,3289,0x12c194011c00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2748,245_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op21,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op21,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3290,3537,0x12c24ee15600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2748,246_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op21,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op21,kernel_graph0,{3535},3535,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3293,3317,0x12c201213c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2749,247_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op9,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op9,kernel_graph0,{3298-3315},3315,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3294,3298,0x12c194011c00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2749,247_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op9,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op9,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3300,3307,0x12c1b4012400,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2752,249_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op19,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op19,kernel_graph0,{3305},3305,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3301,3304,0x12c194011c00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2752,249_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op19,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op19,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3305,3531,0x12c1c9812a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2753,250_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op19,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op19,kernel_graph0,{3529},3529,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3308,3318,0x12c1a4012200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2753,251_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op10,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op10,kernel_graph0,{3313-3315},3315,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3309,3313,0x12c194011c00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2753,251_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op10,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op10,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3315,3321,0x12c1e9c13000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2757,253_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/AddN-op0,kernel_graph0,{3319},3319,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3319,3329,0x12c1f3213400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2757,254_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op8,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op8,kernel_graph0,{3327},3327,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3322,3330,0x12c1d4412c00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2759,255_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op2,kernel_graph0,{3327},3327,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3325,3335,0x12c1a4012200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2760,256_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op3,kernel_graph0,{3333},3333,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3326,3564,0x12c18792a600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2760,256_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op3,kernel_graph0,{3562},3562,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3327,3332,0x12c194011c00,0,DefaultEnhancedAscendMemoryPool,16778752,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2760,256_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op3,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3333,3338,0x12c201213c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2760,257_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op3,kernel_graph0,{3336},3336,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3336,3501,0x12c1e9c13000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2761,258_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/AddN-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/AddN-op1,kernel_graph0,{3339-3342-3350-3498},3498,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3341,3348,0x12c259a15800,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2762,260_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op18,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op18,kernel_graph0,{3346},3346,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3342,3345,0x12c1edc13200,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2762,260_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op18,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op18,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3346,3528,0x12c25ba15a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2764,261_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op18,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op18,kernel_graph0,{3526},3526,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3349,3358,0x12c23d615400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2765,262_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op11,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op11,kernel_graph0,{3353-3356},3356,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3350,3353,0x12c1edc13200,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2765,262_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op11,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op11,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3355,3369,0x12c1f3213400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2767,264_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op4,kernel_graph0,{3364},3364,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3356,3359,0x12c180010400,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2767,264_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op4,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3360,3382,0x12c1f7213600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2768,265_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,kernel_graph0,{3380},3380,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3361,3379,0x12c23d615400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2768,265_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,kernel_graph0,{3377},3377,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3362,3386,0x12c201213c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2768,265_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,kernel_graph0,{3384},3384,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3363,3375,0x12c18000fa00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2768,265_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3364,3376,0x12c20f214400,0,DefaultEnhancedAscendMemoryPool,515969024,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2768,265_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3377,3393,0x12c18e011800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2768,266_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op11,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op11,kernel_graph0,{3389-3391},3391,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3380,3416,0x12c1d4412c00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2771,267_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op9,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op9,kernel_graph0,{3410-3413},3413,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3383,3443,0x12c205213e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2772,268_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op7,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op7,kernel_graph0,{3387-3434-3441},3441,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3384,3387,0x12c180010400,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2772,268_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op7,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op7,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3389,3399,0x12c1f3213400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2774,270_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op10,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op10,kernel_graph0,{3397},3397,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3391,3396,0x12c20f214400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2775,271_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op11,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op11,kernel_graph0,{3394},3394,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3394,3400,0x12c18e011800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2775,272_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op3,kernel_graph0,{3397},3397,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3397,3403,0x12c20f214400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2776,273_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op1,kernel_graph0,{3401},3401,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3401,3407,0x12c201213c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2777,274_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op12,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op12,kernel_graph0,{3405},3405,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3404,3457,0x12c23d615400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2777,275_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op6,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op6,kernel_graph0,{3408-3448-3455},3455,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3405,3408,0x12c180010400,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2777,275_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op6,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op6,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3410,3422,0x12c18e011800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2780,277_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op8,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op8,kernel_graph0,{3420},3420,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3413,3419,0x12c1f3213400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2780,278_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op9,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op9,kernel_graph0,{3417},3417,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3417,3423,0x12c1d4412c00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2782,279_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op2,kernel_graph0,{3420},3420,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3420,3426,0x12c1f3213400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2782,280_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op0,kernel_graph0,{3424},3424,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3424,3430,0x12c201213c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2783,281_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op10,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op10,kernel_graph0,{3428},3428,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3427,3472,0x12c18e011800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2783,282_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op5,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op5,kernel_graph0,{3431-3462-3470},3470,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3428,3431,0x12c180010400,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2783,282_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op5,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op5,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3433,3439,0x12c209214000,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2787,284_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op17,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op17,kernel_graph0,{3437},3437,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3434,3436,0x12c1edc13200,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2787,284_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op17,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op17,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3437,3525,0x12c201213c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2787,285_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op17,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op17,kernel_graph0,{3523},3523,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3440,3480,0x12c1f3213400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2789,286_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op14,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op14,kernel_graph0,{3445-3476},3476,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3441,3445,0x12c1edc13200,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2789,286_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op14,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op14,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3447,3453,0x12c259a15800,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2790,288_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op16,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op16,kernel_graph0,{3451},3451,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3448,3450,0x12c1edc13200,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2790,288_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op16,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op16,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3451,3522,0x12c1f7213600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2792,289_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op16,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op16,kernel_graph0,{3520},3520,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3454,3479,0x12c205213e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2793,290_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op13,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op13,kernel_graph0,{3459-3476},3476,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3455,3459,0x12c1edc13200,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2793,290_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op13,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op13,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3461,3468,0x12c1fb213800,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2794,292_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op15,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op15,kernel_graph0,{3466},3466,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3462,3465,0x12c1edc13200,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2794,292_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op15,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op15,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3466,3519,0x12c23d615400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2795,293_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op15,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op15,kernel_graph0,{3517},3517,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3469,3478,0x12c1800a1800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2797,294_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op12,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op12,kernel_graph0,{3474-3476},3476,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3470,3474,0x12c1edc13200,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2797,294_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op12,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op12,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3476,3483,0x12c1d4412c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2798,296_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/AddN-op0,kernel_graph0,{3481},3481,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3481,3491,0x12c20f214400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2803,297_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op13,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op13,kernel_graph0,{3489},3489,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3484,3492,0x12c1d4412c00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2805,298_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op2,kernel_graph0,{3489},3489,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3487,3497,0x12c217214600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2806,299_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op4,kernel_graph0,{3495},3495,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3488,3567,0x12c180015400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2806,299_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op4,kernel_graph0,{3565},3565,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3489,3494,0x12c1edc13200,0,DefaultEnhancedAscendMemoryPool,16778752,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2806,299_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op4,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3495,3500,0x12c1f3213400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2806,300_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op4,kernel_graph0,{3498},3498,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3498,3504,0x12c205213e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2808,301_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/AddN-op0,kernel_graph0,{3502},3502,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3502,3507,0x12c1d4412c00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2809,302_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op14,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op14,kernel_graph0,{3505},3505,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3505,3516,0x12c20f214400,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2810,303_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/tok_embeddings-LlamaEmbedding/Grad_Gather/UnsortedSegmentSum-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/tok_embeddings-LlamaEmbedding/Grad_Gather/UnsortedSegmentSum-op0,kernel_graph0,{3514},3514,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3509,3513,0x12c18000fa00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2810,304_Default/Mul-op1,Default/Mul-op1,kernel_graph0,{3511},3511,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3511,3577,0x12c18000fc00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2811,305_Default/Reciprocal-op0,Default/Reciprocal-op0,kernel_graph0,{3514-3517-3520-3523-3526-3529-3532-3535-3538-3541-3544-3547-3550-3553-3556-3559-3562-3565-3568-3571-3574},3574,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3514,3812,0x12c187936a00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2812,306_Default/Mul-op2,Default/Mul-op2,kernel_graph0,{3578-3738-3810},3810,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3517,3815,0x12c1f3213400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2813,307_Default/Mul-op3,Default/Mul-op3,kernel_graph0,{3586-3738-3813},3813,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3520,3818,0x12c1e9c13000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2814,308_Default/Mul-op4,Default/Mul-op4,kernel_graph0,{3594-3738-3816},3816,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3523,3821,0x12c1f7213600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2814,309_Default/Mul-op5,Default/Mul-op5,kernel_graph0,{3602-3738-3819},3819,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3526,3824,0x12c23d615400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2816,310_Default/Mul-op6,Default/Mul-op6,kernel_graph0,{3610-3738-3822},3822,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3529,3827,0x12c1d4412c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2816,311_Default/Mul-op7,Default/Mul-op7,kernel_graph0,{3618-3738-3825},3825,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3532,3830,0x12c20f214400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2817,312_Default/Mul-op8,Default/Mul-op8,kernel_graph0,{3626-3738-3828},3828,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3535,3833,0x12c1df012e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2818,313_Default/Mul-op9,Default/Mul-op9,kernel_graph0,{3634-3738-3831},3831,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3538,3836,0x12c180019600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2818,314_Default/Mul-op10,Default/Mul-op10,kernel_graph0,{3642-3738-3834},3834,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3541,3839,0x12c1edc13200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2819,315_Default/Mul-op11,Default/Mul-op11,kernel_graph0,{3650-3738-3837},3837,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3544,3842,0x12c239615200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2820,316_Default/Mul-op12,Default/Mul-op12,kernel_graph0,{3658-3738-3840},3840,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3547,3845,0x12c1fb213800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2821,317_Default/Mul-op13,Default/Mul-op13,kernel_graph0,{3666-3738-3843},3843,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3550,3848,0x12c1ff213a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2821,318_Default/Mul-op14,Default/Mul-op14,kernel_graph0,{3674-3738-3846},3846,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3553,3851,0x12c219e14600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2823,319_Default/Mul-op15,Default/Mul-op15,kernel_graph0,{3682-3738-3849},3849,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3556,3854,0x12c241615600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2824,320_Default/Mul-op16,Default/Mul-op16,kernel_graph0,{3690-3738-3852},3852,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3559,3857,0x12c24c215800,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2824,321_Default/Mul-op17,Default/Mul-op17,kernel_graph0,{3698-3738-3855},3855,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3562,3860,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2827,322_Default/Mul-op18,Default/Mul-op18,kernel_graph0,{3706-3738-3858},3858,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3565,3863,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2827,323_Default/Mul-op19,Default/Mul-op19,kernel_graph0,{3714-3738-3861},3861,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3568,3866,0x12c180015400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2828,324_Default/Mul-op20,Default/Mul-op20,kernel_graph0,{3722-3738-3864},3864,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3571,3869,0x12c180011200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2828,325_Default/Mul-op21,Default/Mul-op21,kernel_graph0,{3730-3738-3867},3867,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3574,3872,0x12c1f1c13400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2830,326_Default/Mul-op22,Default/Mul-op22,kernel_graph0,{3738-3740-3870},3870,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3578,3583,0x12c1a6d36c00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2831,327_Default/clip_grad_norm-ClipGradNorm/Square-op0,Default/clip_grad_norm-ClipGradNorm/Square-op0,kernel_graph0,{3581},3581,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3580,3750,0x12c18000fa00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2832,328_Default/clip_grad_norm-ClipGradNorm/SumExt-op0,Default/clip_grad_norm-ClipGradNorm/SumExt-op0,kernel_graph0,{3584-3748},3748,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3581,3584,0x12c18000b600,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2832,328_Default/clip_grad_norm-ClipGradNorm/SumExt-op0,Default/clip_grad_norm-ClipGradNorm/SumExt-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3586,3591,0x12c209e13c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2833,330_Default/clip_grad_norm-ClipGradNorm/Square-op1,Default/clip_grad_norm-ClipGradNorm/Square-op1,kernel_graph0,{3589},3589,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3588,3751,0x12c18000fc00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2834,331_Default/clip_grad_norm-ClipGradNorm/SumExt-op1,Default/clip_grad_norm-ClipGradNorm/SumExt-op1,kernel_graph0,{3592-3748},3748,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3589,3592,0x12c18000b600,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2834,331_Default/clip_grad_norm-ClipGradNorm/SumExt-op1,Default/clip_grad_norm-ClipGradNorm/SumExt-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3594,3599,0x12c209e13c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2836,333_Default/clip_grad_norm-ClipGradNorm/Square-op2,Default/clip_grad_norm-ClipGradNorm/Square-op2,kernel_graph0,{3597},3597,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3596,3752,0x12c18000fe00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2836,334_Default/clip_grad_norm-ClipGradNorm/SumExt-op2,Default/clip_grad_norm-ClipGradNorm/SumExt-op2,kernel_graph0,{3600-3748},3748,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3597,3600,0x12c18000b600,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2836,334_Default/clip_grad_norm-ClipGradNorm/SumExt-op2,Default/clip_grad_norm-ClipGradNorm/SumExt-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3602,3607,0x12c209e13c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2839,336_Default/clip_grad_norm-ClipGradNorm/Square-op3,Default/clip_grad_norm-ClipGradNorm/Square-op3,kernel_graph0,{3605},3605,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3604,3753,0x12c180010400,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2839,337_Default/clip_grad_norm-ClipGradNorm/SumExt-op3,Default/clip_grad_norm-ClipGradNorm/SumExt-op3,kernel_graph0,{3608-3748},3748,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3605,3608,0x12c18000b600,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2839,337_Default/clip_grad_norm-ClipGradNorm/SumExt-op3,Default/clip_grad_norm-ClipGradNorm/SumExt-op3,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3610,3615,0x12c209e13c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2841,339_Default/clip_grad_norm-ClipGradNorm/Square-op4,Default/clip_grad_norm-ClipGradNorm/Square-op4,kernel_graph0,{3613},3613,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3612,3754,0x12c180010600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2842,340_Default/clip_grad_norm-ClipGradNorm/SumExt-op4,Default/clip_grad_norm-ClipGradNorm/SumExt-op4,kernel_graph0,{3616-3748},3748,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3613,3616,0x12c18000b600,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2842,340_Default/clip_grad_norm-ClipGradNorm/SumExt-op4,Default/clip_grad_norm-ClipGradNorm/SumExt-op4,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3618,3623,0x12c224a14800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2843,342_Default/clip_grad_norm-ClipGradNorm/Square-op5,Default/clip_grad_norm-ClipGradNorm/Square-op5,kernel_graph0,{3621},3621,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3620,3755,0x12c180010800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2845,343_Default/clip_grad_norm-ClipGradNorm/SumExt-op5,Default/clip_grad_norm-ClipGradNorm/SumExt-op5,kernel_graph0,{3624-3748},3748,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3621,3624,0x12c18000b600,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2845,343_Default/clip_grad_norm-ClipGradNorm/SumExt-op5,Default/clip_grad_norm-ClipGradNorm/SumExt-op5,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3626,3631,0x12c224a14800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2846,345_Default/clip_grad_norm-ClipGradNorm/Square-op6,Default/clip_grad_norm-ClipGradNorm/Square-op6,kernel_graph0,{3629},3629,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3628,3756,0x12c180010a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2848,346_Default/clip_grad_norm-ClipGradNorm/SumExt-op6,Default/clip_grad_norm-ClipGradNorm/SumExt-op6,kernel_graph0,{3632-3748},3748,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3629,3632,0x12c18000b600,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2848,346_Default/clip_grad_norm-ClipGradNorm/SumExt-op6,Default/clip_grad_norm-ClipGradNorm/SumExt-op6,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3634,3639,0x12c224a14800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2849,348_Default/clip_grad_norm-ClipGradNorm/Square-op7,Default/clip_grad_norm-ClipGradNorm/Square-op7,kernel_graph0,{3637},3637,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3636,3757,0x12c180010c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2849,349_Default/clip_grad_norm-ClipGradNorm/SumExt-op7,Default/clip_grad_norm-ClipGradNorm/SumExt-op7,kernel_graph0,{3640-3748},3748,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3637,3640,0x12c18000b600,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2849,349_Default/clip_grad_norm-ClipGradNorm/SumExt-op7,Default/clip_grad_norm-ClipGradNorm/SumExt-op7,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3642,3647,0x12c209e13c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2851,351_Default/clip_grad_norm-ClipGradNorm/Square-op8,Default/clip_grad_norm-ClipGradNorm/Square-op8,kernel_graph0,{3645},3645,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3644,3758,0x12c180010e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2852,352_Default/clip_grad_norm-ClipGradNorm/SumExt-op8,Default/clip_grad_norm-ClipGradNorm/SumExt-op8,kernel_graph0,{3648-3748},3748,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3645,3648,0x12c18000b600,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2852,352_Default/clip_grad_norm-ClipGradNorm/SumExt-op8,Default/clip_grad_norm-ClipGradNorm/SumExt-op8,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3650,3655,0x12c209e13c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2853,354_Default/clip_grad_norm-ClipGradNorm/Square-op9,Default/clip_grad_norm-ClipGradNorm/Square-op9,kernel_graph0,{3653},3653,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3652,3759,0x12c180008a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2854,355_Default/clip_grad_norm-ClipGradNorm/SumExt-op9,Default/clip_grad_norm-ClipGradNorm/SumExt-op9,kernel_graph0,{3656-3748},3748,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3653,3656,0x12c18000b600,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2854,355_Default/clip_grad_norm-ClipGradNorm/SumExt-op9,Default/clip_grad_norm-ClipGradNorm/SumExt-op9,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3658,3663,0x12c209e13c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2857,357_Default/clip_grad_norm-ClipGradNorm/Square-op10,Default/clip_grad_norm-ClipGradNorm/Square-op10,kernel_graph0,{3661},3661,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3660,3760,0x12c180008c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2857,358_Default/clip_grad_norm-ClipGradNorm/SumExt-op10,Default/clip_grad_norm-ClipGradNorm/SumExt-op10,kernel_graph0,{3664-3748},3748,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3661,3664,0x12c18000b600,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2857,358_Default/clip_grad_norm-ClipGradNorm/SumExt-op10,Default/clip_grad_norm-ClipGradNorm/SumExt-op10,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3666,3671,0x12c209e13c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2859,360_Default/clip_grad_norm-ClipGradNorm/Square-op11,Default/clip_grad_norm-ClipGradNorm/Square-op11,kernel_graph0,{3669},3669,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3668,3761,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2859,361_Default/clip_grad_norm-ClipGradNorm/SumExt-op11,Default/clip_grad_norm-ClipGradNorm/SumExt-op11,kernel_graph0,{3672-3748},3748,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3669,3672,0x12c18000b600,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2859,361_Default/clip_grad_norm-ClipGradNorm/SumExt-op11,Default/clip_grad_norm-ClipGradNorm/SumExt-op11,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3674,3679,0x12c224a14800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2861,363_Default/clip_grad_norm-ClipGradNorm/Square-op12,Default/clip_grad_norm-ClipGradNorm/Square-op12,kernel_graph0,{3677},3677,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3676,3762,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2862,364_Default/clip_grad_norm-ClipGradNorm/SumExt-op12,Default/clip_grad_norm-ClipGradNorm/SumExt-op12,kernel_graph0,{3680-3748},3748,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3677,3680,0x12c18000b600,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2862,364_Default/clip_grad_norm-ClipGradNorm/SumExt-op12,Default/clip_grad_norm-ClipGradNorm/SumExt-op12,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3682,3687,0x12c224a14800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2864,366_Default/clip_grad_norm-ClipGradNorm/Square-op13,Default/clip_grad_norm-ClipGradNorm/Square-op13,kernel_graph0,{3685},3685,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3684,3763,0x12c180009200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2865,367_Default/clip_grad_norm-ClipGradNorm/SumExt-op13,Default/clip_grad_norm-ClipGradNorm/SumExt-op13,kernel_graph0,{3688-3748},3748,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3685,3688,0x12c18000b600,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2865,367_Default/clip_grad_norm-ClipGradNorm/SumExt-op13,Default/clip_grad_norm-ClipGradNorm/SumExt-op13,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3690,3695,0x12c224a14800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2868,369_Default/clip_grad_norm-ClipGradNorm/Square-op14,Default/clip_grad_norm-ClipGradNorm/Square-op14,kernel_graph0,{3693},3693,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3692,3764,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2868,370_Default/clip_grad_norm-ClipGradNorm/SumExt-op14,Default/clip_grad_norm-ClipGradNorm/SumExt-op14,kernel_graph0,{3696-3748},3748,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3693,3696,0x12c18000b600,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2868,370_Default/clip_grad_norm-ClipGradNorm/SumExt-op14,Default/clip_grad_norm-ClipGradNorm/SumExt-op14,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3698,3703,0x12c1a6d36c00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2872,372_Default/clip_grad_norm-ClipGradNorm/Square-op15,Default/clip_grad_norm-ClipGradNorm/Square-op15,kernel_graph0,{3701},3701,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3700,3765,0x12c180009600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2872,373_Default/clip_grad_norm-ClipGradNorm/SumExt-op15,Default/clip_grad_norm-ClipGradNorm/SumExt-op15,kernel_graph0,{3704-3748},3748,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3701,3704,0x12c18000b600,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2872,373_Default/clip_grad_norm-ClipGradNorm/SumExt-op15,Default/clip_grad_norm-ClipGradNorm/SumExt-op15,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3706,3711,0x12c18000b600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2874,375_Default/clip_grad_norm-ClipGradNorm/Square-op16,Default/clip_grad_norm-ClipGradNorm/Square-op16,kernel_graph0,{3709},3709,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3708,3766,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2875,376_Default/clip_grad_norm-ClipGradNorm/SumExt-op16,Default/clip_grad_norm-ClipGradNorm/SumExt-op16,kernel_graph0,{3712-3748},3748,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3709,3712,0x12c180009a00,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2875,376_Default/clip_grad_norm-ClipGradNorm/SumExt-op16,Default/clip_grad_norm-ClipGradNorm/SumExt-op16,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3714,3719,0x12c18000b600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2877,378_Default/clip_grad_norm-ClipGradNorm/Square-op17,Default/clip_grad_norm-ClipGradNorm/Square-op17,kernel_graph0,{3717},3717,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3716,3767,0x12c180009a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2877,379_Default/clip_grad_norm-ClipGradNorm/SumExt-op17,Default/clip_grad_norm-ClipGradNorm/SumExt-op17,kernel_graph0,{3720-3748},3748,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3717,3720,0x12c180009c00,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2877,379_Default/clip_grad_norm-ClipGradNorm/SumExt-op17,Default/clip_grad_norm-ClipGradNorm/SumExt-op17,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3722,3727,0x12c18000b600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2880,381_Default/clip_grad_norm-ClipGradNorm/Square-op18,Default/clip_grad_norm-ClipGradNorm/Square-op18,kernel_graph0,{3725},3725,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3724,3768,0x12c180009c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2880,382_Default/clip_grad_norm-ClipGradNorm/SumExt-op18,Default/clip_grad_norm-ClipGradNorm/SumExt-op18,kernel_graph0,{3728-3748},3748,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3725,3728,0x12c180009e00,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2880,382_Default/clip_grad_norm-ClipGradNorm/SumExt-op18,Default/clip_grad_norm-ClipGradNorm/SumExt-op18,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3730,3735,0x12c18000b600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2883,384_Default/clip_grad_norm-ClipGradNorm/Square-op19,Default/clip_grad_norm-ClipGradNorm/Square-op19,kernel_graph0,{3733},3733,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3732,3769,0x12c180009e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2883,385_Default/clip_grad_norm-ClipGradNorm/SumExt-op19,Default/clip_grad_norm-ClipGradNorm/SumExt-op19,kernel_graph0,{3736-3748},3748,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3733,3736,0x12c18000a000,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2883,385_Default/clip_grad_norm-ClipGradNorm/SumExt-op19,Default/clip_grad_norm-ClipGradNorm/SumExt-op19,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3738,7103,0x12c18000a000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2887,387_Default/AllFinite-op0,Default/AllFinite-op0,kernel_graph0,{3774-3777-3779},3779,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3740,3745,0x12c18000b600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2887,388_Default/clip_grad_norm-ClipGradNorm/Square-op20,Default/clip_grad_norm-ClipGradNorm/Square-op20,kernel_graph0,{3743},3743,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3742,3770,0x12c18000a200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2889,389_Default/clip_grad_norm-ClipGradNorm/SumExt-op20,Default/clip_grad_norm-ClipGradNorm/SumExt-op20,kernel_graph0,{3746-3748},3748,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3743,3746,0x12c18000a400,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2889,389_Default/clip_grad_norm-ClipGradNorm/SumExt-op20,Default/clip_grad_norm-ClipGradNorm/SumExt-op20,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3748,3773,0x12c18000a400,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2890,391_Default/clip_grad_norm-ClipGradNorm/AddN-op0,Default/clip_grad_norm-ClipGradNorm/AddN-op0,kernel_graph0,{3771},3771,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3771,7100,0x12c18000a200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2893,392_Default/clip_grad_norm-ClipGradNorm/Sqrt-op0,Default/clip_grad_norm-ClipGradNorm/Sqrt-op0,kernel_graph0,{3781-3783},3783,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3774,3804,0x12c18000fa00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2894,393_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Select-op1,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Select-op1,kernel_graph0,{3795-3800},3800,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3777,3921,0x12c18000fc00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2894,394_Default/LogicalNot-op0,Default/LogicalNot-op0,kernel_graph0,{3797},3797,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3779,3807,0x12c18000fe00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2895,395_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/LogicalOr-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/LogicalOr-op0,kernel_graph0,{3805},3805,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3781,3785,0x12c18000a400,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2896,396_Default/clip_grad_norm-ClipGradNorm/GreaterEqual-op0,Default/clip_grad_norm-ClipGradNorm/GreaterEqual-op0,kernel_graph0,{3783},3783,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3783,3788,0x12c18000a600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2896,397_Default/clip_grad_norm-ClipGradNorm/Select-op0,Default/clip_grad_norm-ClipGradNorm/Select-op0,kernel_graph0,{3786},3786,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3786,3791,0x12c18000a400,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2897,398_Default/learning_rate-CosineWithWarmUpLR/Add-op0,Default/learning_rate-CosineWithWarmUpLR/Add-op0,kernel_graph0,{3789},3789,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3789,3794,0x12c18000a600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2898,399_Default/clip_grad_norm-ClipGradNorm/Div-op0,Default/clip_grad_norm-ClipGradNorm/Div-op0,kernel_graph0,{3792},3792,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3792,3873,0x12c18000a400,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2898,400_Default/clip_grad_norm-ClipGradNorm/Minimum-op0,Default/clip_grad_norm-ClipGradNorm/Minimum-op0,kernel_graph0,{3810-3813-3816-3819-3822-3825-3828-3831-3834-3837-3840-3843-3846-3849-3852-3855-3858-3861-3864-3867-3870},3870,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3795,3803,0x12c18000a600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2898,401_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Mul-op1,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Mul-op1,kernel_graph0,{3800},3800,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3797,3802,0x12c18000a800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2898,402_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/LogicalAnd-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/LogicalAnd-op0,kernel_graph0,{3800},3800,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3800,3879,0x12c180010000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2899,403_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Select-op2,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Select-op2,kernel_graph0,{3877},3877,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3805,3875,0x12c18000a600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2900,404_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Select-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Select-op0,kernel_graph0,{3873},3873,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3808,3877,0x12c18000fe00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2900,405_Default/learning_rate-CosineWithWarmUpLR/Add-op1,Default/learning_rate-CosineWithWarmUpLR/Add-op1,kernel_graph0,{3875},3875,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3810,4080,0x12c1a6d36c00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2903,406_Default/clip_grad_norm-ClipGradNorm/Mul-op0,Default/clip_grad_norm-ClipGradNorm/Mul-op0,kernel_graph0,{4076-4078},4078,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3813,4070,0x12c209e13c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2904,407_Default/clip_grad_norm-ClipGradNorm/Mul-op1,Default/clip_grad_norm-ClipGradNorm/Mul-op1,kernel_graph0,{4066-4068},4068,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3816,4062,0x12c1f1c17600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2905,408_Default/clip_grad_norm-ClipGradNorm/Mul-op2,Default/clip_grad_norm-ClipGradNorm/Mul-op2,kernel_graph0,{4058-4060},4060,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3819,4054,0x12c1e9c13000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2905,409_Default/clip_grad_norm-ClipGradNorm/Mul-op3,Default/clip_grad_norm-ClipGradNorm/Mul-op3,kernel_graph0,{4050-4052},4052,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3822,4046,0x12c1f5c17800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2905,410_Default/clip_grad_norm-ClipGradNorm/Mul-op4,Default/clip_grad_norm-ClipGradNorm/Mul-op4,kernel_graph0,{4042-4044},4044,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3825,4038,0x12c1c6136e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2905,411_Default/clip_grad_norm-ClipGradNorm/Mul-op5,Default/clip_grad_norm-ClipGradNorm/Mul-op5,kernel_graph0,{4034-4036},4036,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3828,4030,0x12c1d0d37000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2906,412_Default/clip_grad_norm-ClipGradNorm/Mul-op6,Default/clip_grad_norm-ClipGradNorm/Mul-op6,kernel_graph0,{4026-4028},4028,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3831,4022,0x12c20de13e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2908,413_Default/clip_grad_norm-ClipGradNorm/Mul-op7,Default/clip_grad_norm-ClipGradNorm/Mul-op7,kernel_graph0,{4018-4020},4020,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3834,4014,0x12c23d615400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2909,414_Default/clip_grad_norm-ClipGradNorm/Mul-op8,Default/clip_grad_norm-ClipGradNorm/Mul-op8,kernel_graph0,{4010-4012},4012,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3837,4006,0x12c1db937200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2909,415_Default/clip_grad_norm-ClipGradNorm/Mul-op9,Default/clip_grad_norm-ClipGradNorm/Mul-op9,kernel_graph0,{4002-4004},4004,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3840,3998,0x12c1edc13200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2911,416_Default/clip_grad_norm-ClipGradNorm/Mul-op10,Default/clip_grad_norm-ClipGradNorm/Mul-op10,kernel_graph0,{3994-3996},3996,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3843,3990,0x12c1df937400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2912,417_Default/clip_grad_norm-ClipGradNorm/Mul-op11,Default/clip_grad_norm-ClipGradNorm/Mul-op11,kernel_graph0,{3986-3988},3988,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3846,3982,0x12c224a14800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2913,418_Default/clip_grad_norm-ClipGradNorm/Mul-op12,Default/clip_grad_norm-ClipGradNorm/Mul-op12,kernel_graph0,{3978-3980},3980,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3849,3974,0x12c22f614a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2914,419_Default/clip_grad_norm-ClipGradNorm/Mul-op13,Default/clip_grad_norm-ClipGradNorm/Mul-op13,kernel_graph0,{3970-3972},3972,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3852,3966,0x12c218a14000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2914,420_Default/clip_grad_norm-ClipGradNorm/Mul-op14,Default/clip_grad_norm-ClipGradNorm/Mul-op14,kernel_graph0,{3962-3964},3964,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3855,3958,0x12c180019600,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2916,421_Default/clip_grad_norm-ClipGradNorm/Mul-op15,Default/clip_grad_norm-ClipGradNorm/Mul-op15,kernel_graph0,{3954-3956},3956,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3858,3950,0x12c18000b600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2917,422_Default/clip_grad_norm-ClipGradNorm/Mul-op16,Default/clip_grad_norm-ClipGradNorm/Mul-op16,kernel_graph0,{3946-3948},3948,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3861,3942,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2917,423_Default/clip_grad_norm-ClipGradNorm/Mul-op17,Default/clip_grad_norm-ClipGradNorm/Mul-op17,kernel_graph0,{3938-3940},3940,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3864,3934,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2918,424_Default/clip_grad_norm-ClipGradNorm/Mul-op18,Default/clip_grad_norm-ClipGradNorm/Mul-op18,kernel_graph0,{3930-3932},3932,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3867,3926,0x12c180015400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2919,425_Default/clip_grad_norm-ClipGradNorm/Mul-op19,Default/clip_grad_norm-ClipGradNorm/Mul-op19,kernel_graph0,{3922-3924},3924,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3870,4088,0x12c180011200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2919,426_Default/clip_grad_norm-ClipGradNorm/Mul-op20,Default/clip_grad_norm-ClipGradNorm/Mul-op20,kernel_graph0,{4084-4086},4086,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3880,3889,0x12c18000f800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2923,430_Default/learning_rate-CosineWithWarmUpLR/Cast-op0,Default/learning_rate-CosineWithWarmUpLR/Cast-op0,kernel_graph0,{3882-3885-3887},3887,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3882,3884,0x12c18000fa00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,2924,431_Default/learning_rate-CosineWithWarmUpLR/GreaterEqual-op0,Default/learning_rate-CosineWithWarmUpLR/GreaterEqual-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3885,3890,0x12c18000fa00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3884,435_Default/learning_rate-CosineWithWarmUpLR/Greater-op0,Default/learning_rate-CosineWithWarmUpLR/Greater-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3887,3893,0x12c18000fe00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3885,436_Default/TensorMove-op0,Default/TensorMove-op0,kernel_graph0,{3891},3891,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3891,3896,0x12c180010000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3890,440_Default/learning_rate-CosineWithWarmUpLR/Sub-op1,Default/learning_rate-CosineWithWarmUpLR/Sub-op1,kernel_graph0,{3894},3894,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3894,3899,0x12c18000fe00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3891,441_Default/learning_rate-CosineWithWarmUpLR/Div-op1,Default/learning_rate-CosineWithWarmUpLR/Div-op1,kernel_graph0,{3897},3897,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3897,3902,0x12c180010000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3892,442_Default/learning_rate-CosineWithWarmUpLR/Mul-op1,Default/learning_rate-CosineWithWarmUpLR/Mul-op1,kernel_graph0,{3900},3900,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3900,3905,0x12c18000fe00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3892,443_Default/learning_rate-CosineWithWarmUpLR/Cos-op0,Default/learning_rate-CosineWithWarmUpLR/Cos-op0,kernel_graph0,{3903},3903,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3903,3908,0x12c180010000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3894,444_Default/learning_rate-CosineWithWarmUpLR/Add-op3,Default/learning_rate-CosineWithWarmUpLR/Add-op3,kernel_graph0,{3906},3906,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3906,3911,0x12c18000fe00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3895,445_Default/learning_rate-CosineWithWarmUpLR/Mul-op2,Default/learning_rate-CosineWithWarmUpLR/Mul-op2,kernel_graph0,{3909},3909,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3909,3914,0x12c180010000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3895,446_Default/learning_rate-CosineWithWarmUpLR/Maximum-op0,Default/learning_rate-CosineWithWarmUpLR/Maximum-op0,kernel_graph0,{3912},3912,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3912,3917,0x12c18000fe00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3897,447_Default/learning_rate-CosineWithWarmUpLR/Mul-op3,Default/learning_rate-CosineWithWarmUpLR/Mul-op3,kernel_graph0,{3915},3915,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3915,3920,0x12c180010000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3898,448_Default/learning_rate-CosineWithWarmUpLR/Add-op4,Default/learning_rate-CosineWithWarmUpLR/Add-op4,kernel_graph0,{3918},3918,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3918,7101,0x12c18000fe00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3901,451_Default/Reshape-op0,Default/Reshape-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3922,4751,0x12c1f1c13400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3921,453_Default/optimizer-FP32StateAdamWeightDecay/Mul-op96,Default/optimizer-FP32StateAdamWeightDecay/Mul-op96,kernel_graph0,{4748},4748,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3924,3929,0x12c223614200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3922,454_Default/optimizer-FP32StateAdamWeightDecay/Square-op19,Default/optimizer-FP32StateAdamWeightDecay/Square-op19,kernel_graph0,{3927},3927,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3927,4747,0x12c180015400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3923,455_Default/optimizer-FP32StateAdamWeightDecay/Mul-op98,Default/optimizer-FP32StateAdamWeightDecay/Mul-op98,kernel_graph0,{4744},4744,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3930,4715,0x12c223614200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3924,456_Default/optimizer-FP32StateAdamWeightDecay/Mul-op91,Default/optimizer-FP32StateAdamWeightDecay/Mul-op91,kernel_graph0,{4712},4712,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3932,3937,0x12c223618400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3924,457_Default/optimizer-FP32StateAdamWeightDecay/Square-op18,Default/optimizer-FP32StateAdamWeightDecay/Square-op18,kernel_graph0,{3935},3935,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3935,4719,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3925,458_Default/optimizer-FP32StateAdamWeightDecay/Mul-op93,Default/optimizer-FP32StateAdamWeightDecay/Mul-op93,kernel_graph0,{4716},4716,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3938,4683,0x12c223618400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3926,459_Default/optimizer-FP32StateAdamWeightDecay/Mul-op86,Default/optimizer-FP32StateAdamWeightDecay/Mul-op86,kernel_graph0,{4680},4680,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3940,3945,0x12c22361c600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3927,460_Default/optimizer-FP32StateAdamWeightDecay/Square-op17,Default/optimizer-FP32StateAdamWeightDecay/Square-op17,kernel_graph0,{3943},3943,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3943,4687,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3928,461_Default/optimizer-FP32StateAdamWeightDecay/Mul-op88,Default/optimizer-FP32StateAdamWeightDecay/Mul-op88,kernel_graph0,{4684},4684,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3946,4651,0x12c22361c600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3929,462_Default/optimizer-FP32StateAdamWeightDecay/Mul-op81,Default/optimizer-FP32StateAdamWeightDecay/Mul-op81,kernel_graph0,{4648},4648,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3948,3953,0x12c223620800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3930,463_Default/optimizer-FP32StateAdamWeightDecay/Square-op16,Default/optimizer-FP32StateAdamWeightDecay/Square-op16,kernel_graph0,{3951},3951,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3951,4655,0x12c18000b600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3931,464_Default/optimizer-FP32StateAdamWeightDecay/Mul-op83,Default/optimizer-FP32StateAdamWeightDecay/Mul-op83,kernel_graph0,{4652},4652,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3954,4619,0x12c241615600,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3931,465_Default/optimizer-FP32StateAdamWeightDecay/Mul-op76,Default/optimizer-FP32StateAdamWeightDecay/Mul-op76,kernel_graph0,{4616},4616,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3956,3961,0x12c260a15800,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3932,466_Default/optimizer-FP32StateAdamWeightDecay/Square-op15,Default/optimizer-FP32StateAdamWeightDecay/Square-op15,kernel_graph0,{3959},3959,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3959,4623,0x12c180019600,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3933,467_Default/optimizer-FP32StateAdamWeightDecay/Mul-op78,Default/optimizer-FP32StateAdamWeightDecay/Mul-op78,kernel_graph0,{4620},4620,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3962,4587,0x12c1f9c17a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3934,468_Default/optimizer-FP32StateAdamWeightDecay/Mul-op71,Default/optimizer-FP32StateAdamWeightDecay/Mul-op71,kernel_graph0,{4584},4584,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3964,3969,0x12c260a15800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3935,469_Default/optimizer-FP32StateAdamWeightDecay/Square-op14,Default/optimizer-FP32StateAdamWeightDecay/Square-op14,kernel_graph0,{3967},3967,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3967,4591,0x12c218a14000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3936,470_Default/optimizer-FP32StateAdamWeightDecay/Mul-op73,Default/optimizer-FP32StateAdamWeightDecay/Mul-op73,kernel_graph0,{4588},4588,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3970,4555,0x12c260a15800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3937,471_Default/optimizer-FP32StateAdamWeightDecay/Mul-op66,Default/optimizer-FP32StateAdamWeightDecay/Mul-op66,kernel_graph0,{4552},4552,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3972,3977,0x12c26b615a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3938,472_Default/optimizer-FP32StateAdamWeightDecay/Square-op13,Default/optimizer-FP32StateAdamWeightDecay/Square-op13,kernel_graph0,{3975},3975,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3975,4559,0x12c22f614a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3939,473_Default/optimizer-FP32StateAdamWeightDecay/Mul-op68,Default/optimizer-FP32StateAdamWeightDecay/Mul-op68,kernel_graph0,{4556},4556,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3978,4523,0x12c26b615a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3940,474_Default/optimizer-FP32StateAdamWeightDecay/Mul-op61,Default/optimizer-FP32StateAdamWeightDecay/Mul-op61,kernel_graph0,{4520},4520,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3980,3985,0x12c276215c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3940,475_Default/optimizer-FP32StateAdamWeightDecay/Square-op12,Default/optimizer-FP32StateAdamWeightDecay/Square-op12,kernel_graph0,{3983},3983,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3983,4527,0x12c223620800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3941,476_Default/optimizer-FP32StateAdamWeightDecay/Mul-op63,Default/optimizer-FP32StateAdamWeightDecay/Mul-op63,kernel_graph0,{4524},4524,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3986,4491,0x12c204817c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3941,477_Default/optimizer-FP32StateAdamWeightDecay/Mul-op56,Default/optimizer-FP32StateAdamWeightDecay/Mul-op56,kernel_graph0,{4488},4488,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3988,3993,0x12c1e3937600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3943,478_Default/optimizer-FP32StateAdamWeightDecay/Square-op11,Default/optimizer-FP32StateAdamWeightDecay/Square-op11,kernel_graph0,{3991},3991,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3991,4495,0x12c1df937400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3944,479_Default/optimizer-FP32StateAdamWeightDecay/Mul-op58,Default/optimizer-FP32StateAdamWeightDecay/Mul-op58,kernel_graph0,{4492},4492,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3994,4459,0x12c1e3937600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3944,480_Default/optimizer-FP32StateAdamWeightDecay/Mul-op51,Default/optimizer-FP32StateAdamWeightDecay/Mul-op51,kernel_graph0,{4456},4456,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3996,4001,0x12c19f419800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3946,481_Default/optimizer-FP32StateAdamWeightDecay/Square-op10,Default/optimizer-FP32StateAdamWeightDecay/Square-op10,kernel_graph0,{3999},3999,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 3999,4463,0x12c1edc13200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3947,482_Default/optimizer-FP32StateAdamWeightDecay/Mul-op53,Default/optimizer-FP32StateAdamWeightDecay/Mul-op53,kernel_graph0,{4460},4460,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4002,4427,0x12c19f419800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3947,483_Default/optimizer-FP32StateAdamWeightDecay/Mul-op46,Default/optimizer-FP32StateAdamWeightDecay/Mul-op46,kernel_graph0,{4424},4424,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4004,4009,0x12c276215c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3948,484_Default/optimizer-FP32StateAdamWeightDecay/Square-op9,Default/optimizer-FP32StateAdamWeightDecay/Square-op9,kernel_graph0,{4007},4007,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4007,4431,0x12c1db937200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3949,485_Default/optimizer-FP32StateAdamWeightDecay/Mul-op48,Default/optimizer-FP32StateAdamWeightDecay/Mul-op48,kernel_graph0,{4428},4428,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4010,4395,0x12c276215c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3949,486_Default/optimizer-FP32StateAdamWeightDecay/Mul-op41,Default/optimizer-FP32StateAdamWeightDecay/Mul-op41,kernel_graph0,{4392},4392,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4012,4017,0x12c27a215e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3951,487_Default/optimizer-FP32StateAdamWeightDecay/Square-op8,Default/optimizer-FP32StateAdamWeightDecay/Square-op8,kernel_graph0,{4015},4015,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4015,4399,0x12c23a214c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3952,488_Default/optimizer-FP32StateAdamWeightDecay/Mul-op43,Default/optimizer-FP32StateAdamWeightDecay/Mul-op43,kernel_graph0,{4396},4396,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4018,4363,0x12c27a215e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3952,489_Default/optimizer-FP32StateAdamWeightDecay/Mul-op36,Default/optimizer-FP32StateAdamWeightDecay/Mul-op36,kernel_graph0,{4360},4360,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4020,4025,0x12c284e16000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3954,490_Default/optimizer-FP32StateAdamWeightDecay/Square-op7,Default/optimizer-FP32StateAdamWeightDecay/Square-op7,kernel_graph0,{4023},4023,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4023,4367,0x12c20de13e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3955,491_Default/optimizer-FP32StateAdamWeightDecay/Mul-op38,Default/optimizer-FP32StateAdamWeightDecay/Mul-op38,kernel_graph0,{4364},4364,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4026,4331,0x12c284e16000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3955,492_Default/optimizer-FP32StateAdamWeightDecay/Mul-op31,Default/optimizer-FP32StateAdamWeightDecay/Mul-op31,kernel_graph0,{4328},4328,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4028,4033,0x12c28fa16200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3956,493_Default/optimizer-FP32StateAdamWeightDecay/Square-op6,Default/optimizer-FP32StateAdamWeightDecay/Square-op6,kernel_graph0,{4031},4031,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4031,4335,0x12c1d0d37000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3957,494_Default/optimizer-FP32StateAdamWeightDecay/Mul-op33,Default/optimizer-FP32StateAdamWeightDecay/Mul-op33,kernel_graph0,{4332},4332,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4034,4299,0x12c28fa16200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3957,495_Default/optimizer-FP32StateAdamWeightDecay/Mul-op26,Default/optimizer-FP32StateAdamWeightDecay/Mul-op26,kernel_graph0,{4296},4296,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4036,4041,0x12c29a616400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3959,496_Default/optimizer-FP32StateAdamWeightDecay/Square-op5,Default/optimizer-FP32StateAdamWeightDecay/Square-op5,kernel_graph0,{4039},4039,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4039,4303,0x12c1c6136e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3960,497_Default/optimizer-FP32StateAdamWeightDecay/Mul-op28,Default/optimizer-FP32StateAdamWeightDecay/Mul-op28,kernel_graph0,{4300},4300,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4042,4267,0x12c29a616400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3960,498_Default/optimizer-FP32StateAdamWeightDecay/Mul-op21,Default/optimizer-FP32StateAdamWeightDecay/Mul-op21,kernel_graph0,{4264},4264,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4044,4049,0x12c29e616600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3962,499_Default/optimizer-FP32StateAdamWeightDecay/Square-op4,Default/optimizer-FP32StateAdamWeightDecay/Square-op4,kernel_graph0,{4047},4047,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4047,4271,0x12c1f5c17800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3963,500_Default/optimizer-FP32StateAdamWeightDecay/Mul-op23,Default/optimizer-FP32StateAdamWeightDecay/Mul-op23,kernel_graph0,{4268},4268,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4050,4235,0x12c29e616600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3963,501_Default/optimizer-FP32StateAdamWeightDecay/Mul-op16,Default/optimizer-FP32StateAdamWeightDecay/Mul-op16,kernel_graph0,{4232},4232,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4052,4057,0x12c2a2616800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3964,502_Default/optimizer-FP32StateAdamWeightDecay/Square-op3,Default/optimizer-FP32StateAdamWeightDecay/Square-op3,kernel_graph0,{4055},4055,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4055,4239,0x12c1e7937800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3965,503_Default/optimizer-FP32StateAdamWeightDecay/Mul-op18,Default/optimizer-FP32StateAdamWeightDecay/Mul-op18,kernel_graph0,{4236},4236,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4058,4203,0x12c2a2616800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3965,504_Default/optimizer-FP32StateAdamWeightDecay/Mul-op11,Default/optimizer-FP32StateAdamWeightDecay/Mul-op11,kernel_graph0,{4200},4200,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4060,4065,0x12c2a6616a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3967,505_Default/optimizer-FP32StateAdamWeightDecay/Square-op2,Default/optimizer-FP32StateAdamWeightDecay/Square-op2,kernel_graph0,{4063},4063,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4063,4207,0x12c1f1c17600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3968,506_Default/optimizer-FP32StateAdamWeightDecay/Mul-op13,Default/optimizer-FP32StateAdamWeightDecay/Mul-op13,kernel_graph0,{4204},4204,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4066,4171,0x12c2a6616a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3968,507_Default/optimizer-FP32StateAdamWeightDecay/Mul-op6,Default/optimizer-FP32StateAdamWeightDecay/Mul-op6,kernel_graph0,{4168},4168,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4068,4073,0x12c2aa616c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3970,508_Default/optimizer-FP32StateAdamWeightDecay/Square-op1,Default/optimizer-FP32StateAdamWeightDecay/Square-op1,kernel_graph0,{4071},4071,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4071,4175,0x12c208817e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3970,509_Default/optimizer-FP32StateAdamWeightDecay/Mul-op8,Default/optimizer-FP32StateAdamWeightDecay/Mul-op8,kernel_graph0,{4172},4172,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4074,4099,0x12c180010000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3971,510_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Cast-op0,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Cast-op0,kernel_graph0,{4092-4095-4097},4097,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4076,4139,0x12c2aa616c00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3972,511_Default/optimizer-FP32StateAdamWeightDecay/Mul-op1,Default/optimizer-FP32StateAdamWeightDecay/Mul-op1,kernel_graph0,{4136},4136,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4078,4083,0x12c2c9a16e00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3972,512_Default/optimizer-FP32StateAdamWeightDecay/Square-op0,Default/optimizer-FP32StateAdamWeightDecay/Square-op0,kernel_graph0,{4081},4081,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4081,4143,0x12c1a3419a00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3973,513_Default/optimizer-FP32StateAdamWeightDecay/Mul-op3,Default/optimizer-FP32StateAdamWeightDecay/Mul-op3,kernel_graph0,{4140},4140,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4084,4783,0x12c22e220a00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3975,514_Default/optimizer-FP32StateAdamWeightDecay/Mul-op101,Default/optimizer-FP32StateAdamWeightDecay/Mul-op101,kernel_graph0,{4780},4780,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4086,4091,0x12c22e224c00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3975,515_Default/optimizer-FP32StateAdamWeightDecay/Square-op20,Default/optimizer-FP32StateAdamWeightDecay/Square-op20,kernel_graph0,{4089},4089,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4089,4779,0x12c180011200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3976,516_Default/optimizer-FP32StateAdamWeightDecay/Mul-op103,Default/optimizer-FP32StateAdamWeightDecay/Mul-op103,kernel_graph0,{4776},4776,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4092,4094,0x12c18000f800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,3977,518_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/GreaterEqual-op0,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/GreaterEqual-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4095,4100,0x12c18000f800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4094,522_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Greater-op0,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Greater-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4097,4103,0x12c18000fa00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4095,523_Default/TensorMove-op1,Default/TensorMove-op1,kernel_graph0,{4101},4101,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4101,4106,0x12c18000f800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4100,527_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Sub-op1,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Sub-op1,kernel_graph0,{4104},4104,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4104,4109,0x12c180010000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4101,528_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Div-op1,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Div-op1,kernel_graph0,{4107},4107,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4107,4112,0x12c18000f800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4102,529_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Mul-op1,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Mul-op1,kernel_graph0,{4110},4110,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4110,4115,0x12c180010000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4102,530_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Cos-op0,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Cos-op0,kernel_graph0,{4113},4113,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4113,4118,0x12c18000f800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4104,531_Default/learning_rate-CosineWithWarmUpLR/Add-op6,Default/learning_rate-CosineWithWarmUpLR/Add-op6,kernel_graph0,{4116},4116,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4116,4121,0x12c180010000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4105,532_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Mul-op2,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Mul-op2,kernel_graph0,{4119},4119,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4119,4124,0x12c18000f800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4105,533_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Maximum-op0,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Maximum-op0,kernel_graph0,{4122},4122,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4122,4127,0x12c180010000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4106,534_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Mul-op3,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Mul-op3,kernel_graph0,{4125},4125,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4125,4130,0x12c18000f800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4107,535_Default/learning_rate-CosineWithWarmUpLR/Add-op7,Default/learning_rate-CosineWithWarmUpLR/Add-op7,kernel_graph0,{4128},4128,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4128,4794,0x12c180010000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4110,538_Default/optimizer-FP32StateAdamWeightDecay/Reshape-op0,Default/optimizer-FP32StateAdamWeightDecay/Reshape-op0,kernel_graph0,{4152-4184-4216-4248-4280-4312-4344-4376-4408-4440-4472-4504-4536-4568-4600-4632-4664-4696-4728-4760-4792},4792,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4132,4138,0x12c2c9a16e00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4111,540_Default/optimizer-FP32StateAdamWeightDecay/Mul-op0,Default/optimizer-FP32StateAdamWeightDecay/Mul-op0,kernel_graph0,{4136},4136,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4134,4142,0x12c2e8e17000,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4113,541_Default/optimizer-FP32StateAdamWeightDecay/Mul-op2,Default/optimizer-FP32StateAdamWeightDecay/Mul-op2,kernel_graph0,{4140},4140,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4136,4161,0x12c308217200,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4114,542_Default/learning_rate-CosineWithWarmUpLR/Add-op8,Default/learning_rate-CosineWithWarmUpLR/Add-op8,kernel_graph0,{4149-4159},4159,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4140,4159,0x12c2aa616c00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4114,543_Default/learning_rate-CosineWithWarmUpLR/Add-op9,Default/learning_rate-CosineWithWarmUpLR/Add-op9,kernel_graph0,{4144-4157},4157,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4144,4148,0x12c1a3419a00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4116,544_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op0,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op0,kernel_graph0,{4146},4146,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4146,4151,0x12c2c9a16e00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4117,545_Default/learning_rate-CosineWithWarmUpLR/Add-op10,Default/learning_rate-CosineWithWarmUpLR/Add-op10,kernel_graph0,{4149},4149,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4149,4154,0x12c2e8e17000,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4117,546_Default/optimizer-FP32StateAdamWeightDecay/Div-op0,Default/optimizer-FP32StateAdamWeightDecay/Div-op0,kernel_graph0,{4152},4152,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4152,4157,0x12c2c9a16e00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4119,547_Default/optimizer-FP32StateAdamWeightDecay/Mul-op4,Default/optimizer-FP32StateAdamWeightDecay/Mul-op4,kernel_graph0,{4155},4155,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4155,4163,0x12c2e8e17000,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4120,548_Default/optimizer-FP32StateAdamWeightDecay/Sub-op0,Default/optimizer-FP32StateAdamWeightDecay/Sub-op0,kernel_graph0,{4161},4161,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4164,4170,0x12c1a3419a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4123,552_Default/optimizer-FP32StateAdamWeightDecay/Mul-op5,Default/optimizer-FP32StateAdamWeightDecay/Mul-op5,kernel_graph0,{4168},4168,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4166,4174,0x12c1a7419c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4125,553_Default/optimizer-FP32StateAdamWeightDecay/Mul-op7,Default/optimizer-FP32StateAdamWeightDecay/Mul-op7,kernel_graph0,{4172},4172,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4168,4193,0x12c1ab419e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4126,554_Default/learning_rate-CosineWithWarmUpLR/Add-op11,Default/learning_rate-CosineWithWarmUpLR/Add-op11,kernel_graph0,{4181-4191},4191,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4172,4191,0x12c1a3419a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4126,555_Default/learning_rate-CosineWithWarmUpLR/Add-op12,Default/learning_rate-CosineWithWarmUpLR/Add-op12,kernel_graph0,{4176-4189},4189,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4176,4180,0x12c1a7419c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4128,556_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op1,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op1,kernel_graph0,{4178},4178,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4178,4183,0x12c208817e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4129,557_Default/learning_rate-CosineWithWarmUpLR/Add-op13,Default/learning_rate-CosineWithWarmUpLR/Add-op13,kernel_graph0,{4181},4181,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4181,4186,0x12c1a7419c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4130,558_Default/optimizer-FP32StateAdamWeightDecay/Div-op1,Default/optimizer-FP32StateAdamWeightDecay/Div-op1,kernel_graph0,{4184},4184,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4184,4189,0x12c208817e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4130,559_Default/optimizer-FP32StateAdamWeightDecay/Mul-op9,Default/optimizer-FP32StateAdamWeightDecay/Mul-op9,kernel_graph0,{4187},4187,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4187,4195,0x12c1a7419c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4131,560_Default/optimizer-FP32StateAdamWeightDecay/Sub-op1,Default/optimizer-FP32StateAdamWeightDecay/Sub-op1,kernel_graph0,{4193},4193,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4196,4202,0x12c208817e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4132,564_Default/optimizer-FP32StateAdamWeightDecay/Mul-op10,Default/optimizer-FP32StateAdamWeightDecay/Mul-op10,kernel_graph0,{4200},4200,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4198,4206,0x12c1a3419a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4133,565_Default/optimizer-FP32StateAdamWeightDecay/Mul-op12,Default/optimizer-FP32StateAdamWeightDecay/Mul-op12,kernel_graph0,{4204},4204,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4200,4225,0x12c1a7419c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4133,566_Default/learning_rate-CosineWithWarmUpLR/Add-op14,Default/learning_rate-CosineWithWarmUpLR/Add-op14,kernel_graph0,{4213-4223},4223,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4204,4223,0x12c208817e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4134,567_Default/learning_rate-CosineWithWarmUpLR/Add-op15,Default/learning_rate-CosineWithWarmUpLR/Add-op15,kernel_graph0,{4208-4221},4221,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4208,4212,0x12c1a3419a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4135,568_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op2,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op2,kernel_graph0,{4210},4210,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4210,4215,0x12c1f1c17600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4135,569_Default/learning_rate-CosineWithWarmUpLR/Add-op16,Default/learning_rate-CosineWithWarmUpLR/Add-op16,kernel_graph0,{4213},4213,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4213,4218,0x12c1a3419a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4136,570_Default/optimizer-FP32StateAdamWeightDecay/Div-op2,Default/optimizer-FP32StateAdamWeightDecay/Div-op2,kernel_graph0,{4216},4216,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4216,4221,0x12c1f1c17600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4137,571_Default/optimizer-FP32StateAdamWeightDecay/Mul-op14,Default/optimizer-FP32StateAdamWeightDecay/Mul-op14,kernel_graph0,{4219},4219,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4219,4227,0x12c1a3419a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4137,572_Default/optimizer-FP32StateAdamWeightDecay/Sub-op2,Default/optimizer-FP32StateAdamWeightDecay/Sub-op2,kernel_graph0,{4225},4225,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4228,4234,0x12c1f1c17600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4144,576_Default/optimizer-FP32StateAdamWeightDecay/Mul-op15,Default/optimizer-FP32StateAdamWeightDecay/Mul-op15,kernel_graph0,{4232},4232,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4230,4238,0x12c208817e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4145,577_Default/optimizer-FP32StateAdamWeightDecay/Mul-op17,Default/optimizer-FP32StateAdamWeightDecay/Mul-op17,kernel_graph0,{4236},4236,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4232,4257,0x12c1a3419a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4145,578_Default/learning_rate-CosineWithWarmUpLR/Add-op17,Default/learning_rate-CosineWithWarmUpLR/Add-op17,kernel_graph0,{4245-4255},4255,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4236,4255,0x12c1f1c17600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4146,579_Default/learning_rate-CosineWithWarmUpLR/Add-op18,Default/learning_rate-CosineWithWarmUpLR/Add-op18,kernel_graph0,{4240-4253},4253,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4240,4244,0x12c208817e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4147,580_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op3,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op3,kernel_graph0,{4242},4242,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4242,4247,0x12c1e7937800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4147,581_Default/learning_rate-CosineWithWarmUpLR/Add-op19,Default/learning_rate-CosineWithWarmUpLR/Add-op19,kernel_graph0,{4245},4245,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4245,4250,0x12c208817e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4149,582_Default/optimizer-FP32StateAdamWeightDecay/Div-op3,Default/optimizer-FP32StateAdamWeightDecay/Div-op3,kernel_graph0,{4248},4248,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4248,4253,0x12c1e7937800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4150,583_Default/optimizer-FP32StateAdamWeightDecay/Mul-op19,Default/optimizer-FP32StateAdamWeightDecay/Mul-op19,kernel_graph0,{4251},4251,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4251,4259,0x12c208817e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4150,584_Default/optimizer-FP32StateAdamWeightDecay/Sub-op3,Default/optimizer-FP32StateAdamWeightDecay/Sub-op3,kernel_graph0,{4257},4257,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4260,4266,0x12c1f1c17600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4155,588_Default/optimizer-FP32StateAdamWeightDecay/Mul-op20,Default/optimizer-FP32StateAdamWeightDecay/Mul-op20,kernel_graph0,{4264},4264,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4262,4270,0x12c208817e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4156,589_Default/optimizer-FP32StateAdamWeightDecay/Mul-op22,Default/optimizer-FP32StateAdamWeightDecay/Mul-op22,kernel_graph0,{4268},4268,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4264,4289,0x12c1e7937800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4156,590_Default/learning_rate-CosineWithWarmUpLR/Add-op20,Default/learning_rate-CosineWithWarmUpLR/Add-op20,kernel_graph0,{4277-4287},4287,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4268,4287,0x12c1f1c17600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4157,591_Default/learning_rate-CosineWithWarmUpLR/Add-op21,Default/learning_rate-CosineWithWarmUpLR/Add-op21,kernel_graph0,{4272-4285},4285,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4272,4276,0x12c1f5c17800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4158,592_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op4,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op4,kernel_graph0,{4274},4274,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4274,4279,0x12c208817e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4158,593_Default/learning_rate-CosineWithWarmUpLR/Add-op22,Default/learning_rate-CosineWithWarmUpLR/Add-op22,kernel_graph0,{4277},4277,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4277,4282,0x12c1f5c17800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4158,594_Default/optimizer-FP32StateAdamWeightDecay/Div-op4,Default/optimizer-FP32StateAdamWeightDecay/Div-op4,kernel_graph0,{4280},4280,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4280,4285,0x12c208817e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4158,595_Default/optimizer-FP32StateAdamWeightDecay/Mul-op24,Default/optimizer-FP32StateAdamWeightDecay/Mul-op24,kernel_graph0,{4283},4283,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4283,4291,0x12c1f5c17800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4159,596_Default/optimizer-FP32StateAdamWeightDecay/Sub-op4,Default/optimizer-FP32StateAdamWeightDecay/Sub-op4,kernel_graph0,{4289},4289,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4292,4298,0x12c1a3419a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4161,600_Default/optimizer-FP32StateAdamWeightDecay/Mul-op25,Default/optimizer-FP32StateAdamWeightDecay/Mul-op25,kernel_graph0,{4296},4296,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4294,4302,0x12c1ae019c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4162,601_Default/optimizer-FP32StateAdamWeightDecay/Mul-op27,Default/optimizer-FP32StateAdamWeightDecay/Mul-op27,kernel_graph0,{4300},4300,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4296,4321,0x12c1b8c19e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4162,602_Default/learning_rate-CosineWithWarmUpLR/Add-op23,Default/learning_rate-CosineWithWarmUpLR/Add-op23,kernel_graph0,{4309-4319},4319,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4300,4319,0x12c1a3419a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4162,603_Default/learning_rate-CosineWithWarmUpLR/Add-op24,Default/learning_rate-CosineWithWarmUpLR/Add-op24,kernel_graph0,{4304-4317},4317,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4304,4308,0x12c1ae019c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4162,604_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op5,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op5,kernel_graph0,{4306},4306,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4306,4311,0x12c1c381a000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4164,605_Default/learning_rate-CosineWithWarmUpLR/Add-op25,Default/learning_rate-CosineWithWarmUpLR/Add-op25,kernel_graph0,{4309},4309,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4309,4314,0x12c1ae019c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4165,606_Default/optimizer-FP32StateAdamWeightDecay/Div-op5,Default/optimizer-FP32StateAdamWeightDecay/Div-op5,kernel_graph0,{4312},4312,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4312,4317,0x12c1c381a000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4165,607_Default/optimizer-FP32StateAdamWeightDecay/Mul-op29,Default/optimizer-FP32StateAdamWeightDecay/Mul-op29,kernel_graph0,{4315},4315,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4315,4323,0x12c1ae019c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4166,608_Default/optimizer-FP32StateAdamWeightDecay/Sub-op5,Default/optimizer-FP32StateAdamWeightDecay/Sub-op5,kernel_graph0,{4321},4321,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4324,4330,0x12c1a3419a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4169,612_Default/optimizer-FP32StateAdamWeightDecay/Mul-op30,Default/optimizer-FP32StateAdamWeightDecay/Mul-op30,kernel_graph0,{4328},4328,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4326,4334,0x12c1ae019c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4169,613_Default/optimizer-FP32StateAdamWeightDecay/Mul-op32,Default/optimizer-FP32StateAdamWeightDecay/Mul-op32,kernel_graph0,{4332},4332,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4328,4353,0x12c1b8c19e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4172,614_Default/learning_rate-CosineWithWarmUpLR/Add-op26,Default/learning_rate-CosineWithWarmUpLR/Add-op26,kernel_graph0,{4341-4351},4351,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4332,4351,0x12c1a3419a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4173,615_Default/learning_rate-CosineWithWarmUpLR/Add-op27,Default/learning_rate-CosineWithWarmUpLR/Add-op27,kernel_graph0,{4336-4349},4349,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4336,4340,0x12c1ae019c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4173,616_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op6,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op6,kernel_graph0,{4338},4338,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4338,4343,0x12c1c381a000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4176,617_Default/learning_rate-CosineWithWarmUpLR/Add-op28,Default/learning_rate-CosineWithWarmUpLR/Add-op28,kernel_graph0,{4341},4341,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4341,4346,0x12c1ae019c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4177,618_Default/optimizer-FP32StateAdamWeightDecay/Div-op6,Default/optimizer-FP32StateAdamWeightDecay/Div-op6,kernel_graph0,{4344},4344,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4344,4349,0x12c1c381a000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4177,619_Default/optimizer-FP32StateAdamWeightDecay/Mul-op34,Default/optimizer-FP32StateAdamWeightDecay/Mul-op34,kernel_graph0,{4347},4347,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4347,4355,0x12c1ae019c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4178,620_Default/optimizer-FP32StateAdamWeightDecay/Sub-op6,Default/optimizer-FP32StateAdamWeightDecay/Sub-op6,kernel_graph0,{4353},4353,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4356,4362,0x12c1a3419a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4182,624_Default/optimizer-FP32StateAdamWeightDecay/Mul-op35,Default/optimizer-FP32StateAdamWeightDecay/Mul-op35,kernel_graph0,{4360},4360,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4358,4366,0x12c1ae019c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4182,625_Default/optimizer-FP32StateAdamWeightDecay/Mul-op37,Default/optimizer-FP32StateAdamWeightDecay/Mul-op37,kernel_graph0,{4364},4364,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4360,4385,0x12c1b8c19e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4184,626_Default/learning_rate-CosineWithWarmUpLR/Add-op29,Default/learning_rate-CosineWithWarmUpLR/Add-op29,kernel_graph0,{4373-4383},4383,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4364,4383,0x12c1a3419a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4185,627_Default/learning_rate-CosineWithWarmUpLR/Add-op30,Default/learning_rate-CosineWithWarmUpLR/Add-op30,kernel_graph0,{4368-4381},4381,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4368,4372,0x12c1ae019c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4185,628_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op7,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op7,kernel_graph0,{4370},4370,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4370,4375,0x12c208817e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4187,629_Default/learning_rate-CosineWithWarmUpLR/Add-op31,Default/learning_rate-CosineWithWarmUpLR/Add-op31,kernel_graph0,{4373},4373,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4373,4378,0x12c1ae019c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4188,630_Default/optimizer-FP32StateAdamWeightDecay/Div-op7,Default/optimizer-FP32StateAdamWeightDecay/Div-op7,kernel_graph0,{4376},4376,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4376,4381,0x12c208817e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4188,631_Default/optimizer-FP32StateAdamWeightDecay/Mul-op39,Default/optimizer-FP32StateAdamWeightDecay/Mul-op39,kernel_graph0,{4379},4379,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4379,4387,0x12c1ae019c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4189,632_Default/optimizer-FP32StateAdamWeightDecay/Sub-op7,Default/optimizer-FP32StateAdamWeightDecay/Sub-op7,kernel_graph0,{4385},4385,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4388,4394,0x12c1e7937800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4190,636_Default/optimizer-FP32StateAdamWeightDecay/Mul-op40,Default/optimizer-FP32StateAdamWeightDecay/Mul-op40,kernel_graph0,{4392},4392,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4390,4398,0x12c1f1c17600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4191,637_Default/optimizer-FP32StateAdamWeightDecay/Mul-op42,Default/optimizer-FP32StateAdamWeightDecay/Mul-op42,kernel_graph0,{4396},4396,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4392,4417,0x12c1f5c17800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4192,638_Default/learning_rate-CosineWithWarmUpLR/Add-op32,Default/learning_rate-CosineWithWarmUpLR/Add-op32,kernel_graph0,{4405-4415},4415,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4396,4415,0x12c1e7937800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4192,639_Default/learning_rate-CosineWithWarmUpLR/Add-op33,Default/learning_rate-CosineWithWarmUpLR/Add-op33,kernel_graph0,{4400-4413},4413,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4400,4404,0x12c1f1c17600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4192,640_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op8,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op8,kernel_graph0,{4402},4402,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4402,4407,0x12c23a214c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4193,641_Default/learning_rate-CosineWithWarmUpLR/Add-op34,Default/learning_rate-CosineWithWarmUpLR/Add-op34,kernel_graph0,{4405},4405,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4405,4410,0x12c1f1c17600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4194,642_Default/optimizer-FP32StateAdamWeightDecay/Div-op8,Default/optimizer-FP32StateAdamWeightDecay/Div-op8,kernel_graph0,{4408},4408,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4408,4413,0x12c23a214c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4194,643_Default/optimizer-FP32StateAdamWeightDecay/Mul-op44,Default/optimizer-FP32StateAdamWeightDecay/Mul-op44,kernel_graph0,{4411},4411,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4411,4419,0x12c1f1c17600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4194,644_Default/optimizer-FP32StateAdamWeightDecay/Sub-op8,Default/optimizer-FP32StateAdamWeightDecay/Sub-op8,kernel_graph0,{4417},4417,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4420,4426,0x12c1e7937800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4197,648_Default/optimizer-FP32StateAdamWeightDecay/Mul-op45,Default/optimizer-FP32StateAdamWeightDecay/Mul-op45,kernel_graph0,{4424},4424,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4422,4430,0x12c23a214c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4198,649_Default/optimizer-FP32StateAdamWeightDecay/Mul-op47,Default/optimizer-FP32StateAdamWeightDecay/Mul-op47,kernel_graph0,{4428},4428,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4424,4449,0x12c1f1c17600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4199,650_Default/learning_rate-CosineWithWarmUpLR/Add-op35,Default/learning_rate-CosineWithWarmUpLR/Add-op35,kernel_graph0,{4437-4447},4447,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4428,4447,0x12c1f5c17800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4199,651_Default/learning_rate-CosineWithWarmUpLR/Add-op36,Default/learning_rate-CosineWithWarmUpLR/Add-op36,kernel_graph0,{4432-4445},4445,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4432,4436,0x12c1e7937800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4200,652_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op9,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op9,kernel_graph0,{4434},4434,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4434,4439,0x12c23a214c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4201,653_Default/learning_rate-CosineWithWarmUpLR/Add-op37,Default/learning_rate-CosineWithWarmUpLR/Add-op37,kernel_graph0,{4437},4437,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4437,4442,0x12c1e7937800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4203,654_Default/optimizer-FP32StateAdamWeightDecay/Div-op9,Default/optimizer-FP32StateAdamWeightDecay/Div-op9,kernel_graph0,{4440},4440,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4440,4445,0x12c23a214c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4204,655_Default/optimizer-FP32StateAdamWeightDecay/Mul-op49,Default/optimizer-FP32StateAdamWeightDecay/Mul-op49,kernel_graph0,{4443},4443,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4443,4451,0x12c1e7937800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4205,656_Default/optimizer-FP32StateAdamWeightDecay/Sub-op9,Default/optimizer-FP32StateAdamWeightDecay/Sub-op9,kernel_graph0,{4449},4449,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4452,4458,0x12c1e7937800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4209,660_Default/optimizer-FP32StateAdamWeightDecay/Mul-op50,Default/optimizer-FP32StateAdamWeightDecay/Mul-op50,kernel_graph0,{4456},4456,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4454,4462,0x12c23a214c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4210,661_Default/optimizer-FP32StateAdamWeightDecay/Mul-op52,Default/optimizer-FP32StateAdamWeightDecay/Mul-op52,kernel_graph0,{4460},4460,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4456,4481,0x12c1f1c17600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4211,662_Default/learning_rate-CosineWithWarmUpLR/Add-op38,Default/learning_rate-CosineWithWarmUpLR/Add-op38,kernel_graph0,{4469-4479},4479,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4460,4479,0x12c1f5c17800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4211,663_Default/learning_rate-CosineWithWarmUpLR/Add-op39,Default/learning_rate-CosineWithWarmUpLR/Add-op39,kernel_graph0,{4464-4477},4477,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4464,4468,0x12c23a214c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4213,664_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op10,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op10,kernel_graph0,{4466},4466,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4466,4471,0x12c1e3937600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4214,665_Default/learning_rate-CosineWithWarmUpLR/Add-op40,Default/learning_rate-CosineWithWarmUpLR/Add-op40,kernel_graph0,{4469},4469,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4469,4474,0x12c23a214c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4214,666_Default/optimizer-FP32StateAdamWeightDecay/Div-op10,Default/optimizer-FP32StateAdamWeightDecay/Div-op10,kernel_graph0,{4472},4472,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4472,4477,0x12c1e3937600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4216,667_Default/optimizer-FP32StateAdamWeightDecay/Mul-op54,Default/optimizer-FP32StateAdamWeightDecay/Mul-op54,kernel_graph0,{4475},4475,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4475,4483,0x12c23a214c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4217,668_Default/optimizer-FP32StateAdamWeightDecay/Sub-op10,Default/optimizer-FP32StateAdamWeightDecay/Sub-op10,kernel_graph0,{4481},4481,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4484,4490,0x12c23a214c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4220,672_Default/optimizer-FP32StateAdamWeightDecay/Mul-op55,Default/optimizer-FP32StateAdamWeightDecay/Mul-op55,kernel_graph0,{4488},4488,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4486,4494,0x12c1f1c17600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4221,673_Default/optimizer-FP32StateAdamWeightDecay/Mul-op57,Default/optimizer-FP32StateAdamWeightDecay/Mul-op57,kernel_graph0,{4492},4492,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4488,4513,0x12c1f5c17800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4222,674_Default/learning_rate-CosineWithWarmUpLR/Add-op41,Default/learning_rate-CosineWithWarmUpLR/Add-op41,kernel_graph0,{4501-4511},4511,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4492,4511,0x12c23a214c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4222,675_Default/learning_rate-CosineWithWarmUpLR/Add-op42,Default/learning_rate-CosineWithWarmUpLR/Add-op42,kernel_graph0,{4496-4509},4509,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4496,4500,0x12c1f1c17600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4222,676_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op11,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op11,kernel_graph0,{4498},4498,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4498,4503,0x12c204817c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4223,677_Default/learning_rate-CosineWithWarmUpLR/Add-op43,Default/learning_rate-CosineWithWarmUpLR/Add-op43,kernel_graph0,{4501},4501,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4501,4506,0x12c1f1c17600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4224,678_Default/optimizer-FP32StateAdamWeightDecay/Div-op11,Default/optimizer-FP32StateAdamWeightDecay/Div-op11,kernel_graph0,{4504},4504,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4504,4509,0x12c204817c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4224,679_Default/optimizer-FP32StateAdamWeightDecay/Mul-op59,Default/optimizer-FP32StateAdamWeightDecay/Mul-op59,kernel_graph0,{4507},4507,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4507,4515,0x12c1f1c17600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4224,680_Default/optimizer-FP32StateAdamWeightDecay/Sub-op11,Default/optimizer-FP32StateAdamWeightDecay/Sub-op11,kernel_graph0,{4513},4513,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4516,4522,0x12c204817c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4226,684_Default/optimizer-FP32StateAdamWeightDecay/Mul-op60,Default/optimizer-FP32StateAdamWeightDecay/Mul-op60,kernel_graph0,{4520},4520,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4518,4526,0x12c19f419800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4228,685_Default/optimizer-FP32StateAdamWeightDecay/Mul-op62,Default/optimizer-FP32StateAdamWeightDecay/Mul-op62,kernel_graph0,{4524},4524,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4520,4545,0x12c1aa019a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4228,686_Default/learning_rate-CosineWithWarmUpLR/Add-op44,Default/learning_rate-CosineWithWarmUpLR/Add-op44,kernel_graph0,{4533-4543},4543,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4524,4543,0x12c204817c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4229,687_Default/learning_rate-CosineWithWarmUpLR/Add-op45,Default/learning_rate-CosineWithWarmUpLR/Add-op45,kernel_graph0,{4528-4541},4541,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4528,4532,0x12c19f419800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4229,688_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op12,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op12,kernel_graph0,{4530},4530,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4530,4535,0x12c223620800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4230,689_Default/learning_rate-CosineWithWarmUpLR/Add-op46,Default/learning_rate-CosineWithWarmUpLR/Add-op46,kernel_graph0,{4533},4533,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4533,4538,0x12c19f419800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4231,690_Default/optimizer-FP32StateAdamWeightDecay/Div-op12,Default/optimizer-FP32StateAdamWeightDecay/Div-op12,kernel_graph0,{4536},4536,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4536,4541,0x12c223620800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4231,691_Default/optimizer-FP32StateAdamWeightDecay/Mul-op64,Default/optimizer-FP32StateAdamWeightDecay/Mul-op64,kernel_graph0,{4539},4539,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4539,4547,0x12c19f419800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4232,692_Default/optimizer-FP32StateAdamWeightDecay/Sub-op12,Default/optimizer-FP32StateAdamWeightDecay/Sub-op12,kernel_graph0,{4545},4545,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4548,4554,0x12c223620800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4237,696_Default/optimizer-FP32StateAdamWeightDecay/Mul-op65,Default/optimizer-FP32StateAdamWeightDecay/Mul-op65,kernel_graph0,{4552},4552,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4550,4558,0x12c204817c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4237,697_Default/optimizer-FP32StateAdamWeightDecay/Mul-op67,Default/optimizer-FP32StateAdamWeightDecay/Mul-op67,kernel_graph0,{4556},4556,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4552,4577,0x12c19f419800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4240,698_Default/learning_rate-CosineWithWarmUpLR/Add-op47,Default/learning_rate-CosineWithWarmUpLR/Add-op47,kernel_graph0,{4565-4575},4575,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4556,4575,0x12c223620800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4241,699_Default/learning_rate-CosineWithWarmUpLR/Add-op48,Default/learning_rate-CosineWithWarmUpLR/Add-op48,kernel_graph0,{4560-4573},4573,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4560,4564,0x12c22e224c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4241,700_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op13,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op13,kernel_graph0,{4562},4562,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4562,4567,0x12c204817c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4242,701_Default/learning_rate-CosineWithWarmUpLR/Add-op49,Default/learning_rate-CosineWithWarmUpLR/Add-op49,kernel_graph0,{4565},4565,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4565,4570,0x12c22e224c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4243,702_Default/optimizer-FP32StateAdamWeightDecay/Div-op13,Default/optimizer-FP32StateAdamWeightDecay/Div-op13,kernel_graph0,{4568},4568,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4568,4573,0x12c204817c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4243,703_Default/optimizer-FP32StateAdamWeightDecay/Mul-op69,Default/optimizer-FP32StateAdamWeightDecay/Mul-op69,kernel_graph0,{4571},4571,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4571,4579,0x12c22e224c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4245,704_Default/optimizer-FP32StateAdamWeightDecay/Sub-op13,Default/optimizer-FP32StateAdamWeightDecay/Sub-op13,kernel_graph0,{4577},4577,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4580,4586,0x12c223620800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4248,708_Default/optimizer-FP32StateAdamWeightDecay/Mul-op70,Default/optimizer-FP32StateAdamWeightDecay/Mul-op70,kernel_graph0,{4584},4584,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4582,4590,0x12c22e224c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4249,709_Default/optimizer-FP32StateAdamWeightDecay/Mul-op72,Default/optimizer-FP32StateAdamWeightDecay/Mul-op72,kernel_graph0,{4588},4588,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4584,4609,0x12c204817c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4250,710_Default/learning_rate-CosineWithWarmUpLR/Add-op50,Default/learning_rate-CosineWithWarmUpLR/Add-op50,kernel_graph0,{4597-4607},4607,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4588,4607,0x12c223620800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4251,711_Default/learning_rate-CosineWithWarmUpLR/Add-op51,Default/learning_rate-CosineWithWarmUpLR/Add-op51,kernel_graph0,{4592-4605},4605,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4592,4596,0x12c1f1c17600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4252,712_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op14,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op14,kernel_graph0,{4594},4594,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4594,4599,0x12c22e224c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4253,713_Default/learning_rate-CosineWithWarmUpLR/Add-op52,Default/learning_rate-CosineWithWarmUpLR/Add-op52,kernel_graph0,{4597},4597,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4597,4602,0x12c1f1c17600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4253,714_Default/optimizer-FP32StateAdamWeightDecay/Div-op14,Default/optimizer-FP32StateAdamWeightDecay/Div-op14,kernel_graph0,{4600},4600,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4600,4605,0x12c22e224c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4254,715_Default/optimizer-FP32StateAdamWeightDecay/Mul-op74,Default/optimizer-FP32StateAdamWeightDecay/Mul-op74,kernel_graph0,{4603},4603,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4603,4611,0x12c1f1c17600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4254,716_Default/optimizer-FP32StateAdamWeightDecay/Sub-op14,Default/optimizer-FP32StateAdamWeightDecay/Sub-op14,kernel_graph0,{4609},4609,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4612,4618,0x12c1f1c17600,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4256,720_Default/optimizer-FP32StateAdamWeightDecay/Mul-op75,Default/optimizer-FP32StateAdamWeightDecay/Mul-op75,kernel_graph0,{4616},4616,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4614,4622,0x12c19f419800,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4256,721_Default/optimizer-FP32StateAdamWeightDecay/Mul-op77,Default/optimizer-FP32StateAdamWeightDecay/Mul-op77,kernel_graph0,{4620},4620,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4616,4641,0x12c1be819a00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4257,722_Default/learning_rate-CosineWithWarmUpLR/Add-op53,Default/learning_rate-CosineWithWarmUpLR/Add-op53,kernel_graph0,{4629-4639},4639,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4620,4639,0x12c1f1c17600,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4258,723_Default/learning_rate-CosineWithWarmUpLR/Add-op54,Default/learning_rate-CosineWithWarmUpLR/Add-op54,kernel_graph0,{4624-4637},4637,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4624,4628,0x12c180019600,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4258,724_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op15,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op15,kernel_graph0,{4626},4626,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4626,4631,0x12c19f419800,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4258,725_Default/learning_rate-CosineWithWarmUpLR/Add-op55,Default/learning_rate-CosineWithWarmUpLR/Add-op55,kernel_graph0,{4629},4629,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4629,4634,0x12c180019600,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4260,726_Default/optimizer-FP32StateAdamWeightDecay/Div-op15,Default/optimizer-FP32StateAdamWeightDecay/Div-op15,kernel_graph0,{4632},4632,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4632,4637,0x12c19f419800,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4260,727_Default/optimizer-FP32StateAdamWeightDecay/Mul-op79,Default/optimizer-FP32StateAdamWeightDecay/Mul-op79,kernel_graph0,{4635},4635,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4635,4643,0x12c180019600,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4261,728_Default/optimizer-FP32StateAdamWeightDecay/Sub-op15,Default/optimizer-FP32StateAdamWeightDecay/Sub-op15,kernel_graph0,{4641},4641,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4644,4650,0x12c223620800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4263,732_Default/optimizer-FP32StateAdamWeightDecay/Mul-op80,Default/optimizer-FP32StateAdamWeightDecay/Mul-op80,kernel_graph0,{4648},4648,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4646,4654,0x12c223624a00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4264,733_Default/optimizer-FP32StateAdamWeightDecay/Mul-op82,Default/optimizer-FP32StateAdamWeightDecay/Mul-op82,kernel_graph0,{4652},4652,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4648,4673,0x12c223628c00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4265,734_Default/learning_rate-CosineWithWarmUpLR/Add-op56,Default/learning_rate-CosineWithWarmUpLR/Add-op56,kernel_graph0,{4661-4671},4671,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4652,4671,0x12c22361c600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4267,735_Default/learning_rate-CosineWithWarmUpLR/Add-op57,Default/learning_rate-CosineWithWarmUpLR/Add-op57,kernel_graph0,{4656-4669},4669,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4656,4660,0x12c18000b600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4268,736_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op16,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op16,kernel_graph0,{4658},4658,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4658,4663,0x12c223620800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4269,737_Default/learning_rate-CosineWithWarmUpLR/Add-op58,Default/learning_rate-CosineWithWarmUpLR/Add-op58,kernel_graph0,{4661},4661,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4661,4666,0x12c223624a00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4269,738_Default/optimizer-FP32StateAdamWeightDecay/Div-op16,Default/optimizer-FP32StateAdamWeightDecay/Div-op16,kernel_graph0,{4664},4664,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4664,4669,0x12c223620800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4272,739_Default/optimizer-FP32StateAdamWeightDecay/Mul-op84,Default/optimizer-FP32StateAdamWeightDecay/Mul-op84,kernel_graph0,{4667},4667,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4667,4675,0x12c223624a00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4272,740_Default/optimizer-FP32StateAdamWeightDecay/Sub-op16,Default/optimizer-FP32StateAdamWeightDecay/Sub-op16,kernel_graph0,{4673},4673,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4676,4682,0x12c18000b600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4275,744_Default/optimizer-FP32StateAdamWeightDecay/Mul-op85,Default/optimizer-FP32StateAdamWeightDecay/Mul-op85,kernel_graph0,{4680},4680,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4678,4686,0x12c22361c600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4277,745_Default/optimizer-FP32StateAdamWeightDecay/Mul-op87,Default/optimizer-FP32StateAdamWeightDecay/Mul-op87,kernel_graph0,{4684},4684,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4680,4705,0x12c223620800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4277,746_Default/learning_rate-CosineWithWarmUpLR/Add-op59,Default/learning_rate-CosineWithWarmUpLR/Add-op59,kernel_graph0,{4693-4703},4703,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4684,4703,0x12c223618400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4278,747_Default/learning_rate-CosineWithWarmUpLR/Add-op60,Default/learning_rate-CosineWithWarmUpLR/Add-op60,kernel_graph0,{4688-4701},4701,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4688,4692,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4279,748_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op17,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op17,kernel_graph0,{4690},4690,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4690,4695,0x12c22361c600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4280,749_Default/learning_rate-CosineWithWarmUpLR/Add-op61,Default/learning_rate-CosineWithWarmUpLR/Add-op61,kernel_graph0,{4693},4693,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4693,4698,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4281,750_Default/optimizer-FP32StateAdamWeightDecay/Div-op17,Default/optimizer-FP32StateAdamWeightDecay/Div-op17,kernel_graph0,{4696},4696,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4696,4701,0x12c22361c600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4283,751_Default/optimizer-FP32StateAdamWeightDecay/Mul-op89,Default/optimizer-FP32StateAdamWeightDecay/Mul-op89,kernel_graph0,{4699},4699,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4699,4707,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4283,752_Default/optimizer-FP32StateAdamWeightDecay/Sub-op17,Default/optimizer-FP32StateAdamWeightDecay/Sub-op17,kernel_graph0,{4705},4705,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4708,4714,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4286,756_Default/optimizer-FP32StateAdamWeightDecay/Mul-op90,Default/optimizer-FP32StateAdamWeightDecay/Mul-op90,kernel_graph0,{4712},4712,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4710,4718,0x12c18000b600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4286,757_Default/optimizer-FP32StateAdamWeightDecay/Mul-op92,Default/optimizer-FP32StateAdamWeightDecay/Mul-op92,kernel_graph0,{4716},4716,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4712,4737,0x12c223618400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4286,758_Default/learning_rate-CosineWithWarmUpLR/Add-op62,Default/learning_rate-CosineWithWarmUpLR/Add-op62,kernel_graph0,{4725-4735},4735,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4716,4735,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4287,759_Default/learning_rate-CosineWithWarmUpLR/Add-op63,Default/learning_rate-CosineWithWarmUpLR/Add-op63,kernel_graph0,{4720-4733},4733,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4720,4724,0x12c18000b600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4288,760_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op18,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op18,kernel_graph0,{4722},4722,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4722,4727,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4288,761_Default/learning_rate-CosineWithWarmUpLR/Add-op64,Default/learning_rate-CosineWithWarmUpLR/Add-op64,kernel_graph0,{4725},4725,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4725,4730,0x12c18000b600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4288,762_Default/optimizer-FP32StateAdamWeightDecay/Div-op18,Default/optimizer-FP32StateAdamWeightDecay/Div-op18,kernel_graph0,{4728},4728,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4728,4733,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4289,763_Default/optimizer-FP32StateAdamWeightDecay/Mul-op94,Default/optimizer-FP32StateAdamWeightDecay/Mul-op94,kernel_graph0,{4731},4731,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4731,4739,0x12c18000b600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4290,764_Default/optimizer-FP32StateAdamWeightDecay/Sub-op18,Default/optimizer-FP32StateAdamWeightDecay/Sub-op18,kernel_graph0,{4737},4737,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4740,4746,0x12c18000b600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4292,768_Default/optimizer-FP32StateAdamWeightDecay/Mul-op97,Default/optimizer-FP32StateAdamWeightDecay/Mul-op97,kernel_graph0,{4744},4744,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4742,4750,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4293,769_Default/optimizer-FP32StateAdamWeightDecay/Mul-op95,Default/optimizer-FP32StateAdamWeightDecay/Mul-op95,kernel_graph0,{4748},4748,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4744,4767,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4293,770_Default/learning_rate-CosineWithWarmUpLR/Add-op66,Default/learning_rate-CosineWithWarmUpLR/Add-op66,kernel_graph0,{4752-4765},4765,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4748,4769,0x12c18000b600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4294,771_Default/learning_rate-CosineWithWarmUpLR/Add-op65,Default/learning_rate-CosineWithWarmUpLR/Add-op65,kernel_graph0,{4757-4767},4767,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4752,4756,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4295,772_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op19,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op19,kernel_graph0,{4754},4754,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4754,4759,0x12c180015400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4295,773_Default/learning_rate-CosineWithWarmUpLR/Add-op67,Default/learning_rate-CosineWithWarmUpLR/Add-op67,kernel_graph0,{4757},4757,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4757,4762,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4296,774_Default/optimizer-FP32StateAdamWeightDecay/Div-op19,Default/optimizer-FP32StateAdamWeightDecay/Div-op19,kernel_graph0,{4760},4760,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4760,4765,0x12c180015400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4297,775_Default/optimizer-FP32StateAdamWeightDecay/Mul-op99,Default/optimizer-FP32StateAdamWeightDecay/Mul-op99,kernel_graph0,{4763},4763,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4763,4771,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4299,776_Default/optimizer-FP32StateAdamWeightDecay/Sub-op19,Default/optimizer-FP32StateAdamWeightDecay/Sub-op19,kernel_graph0,{4769},4769,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4772,4778,0x12c18000b600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4304,780_Default/optimizer-FP32StateAdamWeightDecay/Mul-op102,Default/optimizer-FP32StateAdamWeightDecay/Mul-op102,kernel_graph0,{4776},4776,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4774,4782,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4305,781_Default/optimizer-FP32StateAdamWeightDecay/Mul-op100,Default/optimizer-FP32StateAdamWeightDecay/Mul-op100,kernel_graph0,{4780},4780,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4776,4804,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4305,782_Default/learning_rate-CosineWithWarmUpLR/Add-op69,Default/learning_rate-CosineWithWarmUpLR/Add-op69,kernel_graph0,{4784-4802},4802,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4780,4802,0x12c18000b600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4306,783_Default/learning_rate-CosineWithWarmUpLR/Add-op68,Default/learning_rate-CosineWithWarmUpLR/Add-op68,kernel_graph0,{4789-4800},4800,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4784,4788,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4307,784_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op20,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op20,kernel_graph0,{4786},4786,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4786,4791,0x12c180011200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4307,785_Default/learning_rate-CosineWithWarmUpLR/Add-op70,Default/learning_rate-CosineWithWarmUpLR/Add-op70,kernel_graph0,{4789},4789,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4789,4795,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4309,786_Default/optimizer-FP32StateAdamWeightDecay/Div-op20,Default/optimizer-FP32StateAdamWeightDecay/Div-op20,kernel_graph0,{4792},4792,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4792,4798,0x12c180011200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4309,787_Default/optimizer-FP32StateAdamWeightDecay/Mul-op104,Default/optimizer-FP32StateAdamWeightDecay/Mul-op104,kernel_graph0,{4796},4796,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4796,4800,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4310,788_Default/optimizer-FP32StateAdamWeightDecay/Sub-op20,Default/optimizer-FP32StateAdamWeightDecay/Sub-op20,kernel_graph0,{4798},4798,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4805,7104,0x12c180010000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4314,792_Default/TensorMove-op2,Default/TensorMove-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4809,7102,0x12c75d35ee00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/output_actor.cc,689,Other,4807,kernel_graph_0_OutputActor,UpdateOutputDeviceAddress,,{},,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 4815,5219,0x12c75d35f000,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Kernel,4814,0_Default/network-LlamaForCausalLM/StridedSlice-op0,inputs0,0_Default/network-LlamaForCausalLM/StridedSlice-op0,{4816-5217},5217,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,1,0, 4816,5798,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4814,0_Default/network-LlamaForCausalLM/StridedSlice-op0,inputs0,0_Default/network-LlamaForCausalLM/StridedSlice-op0,{4818-4820-5795},5795,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4818,4824,0x12c180010200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4816,1_Default/network-LlamaForCausalLM/model-LlamaModel/tok_embeddings-LlamaEmbedding/Gather-op0,Default/network-LlamaForCausalLM/model-LlamaModel/tok_embeddings-LlamaEmbedding/Gather-op0,kernel_graph0,{4822},4822,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4820,4830,0x12c18000a400,0,DefaultEnhancedAscendMemoryPool,8704,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4817,2_Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/NotEqual-op0,Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/NotEqual-op0,kernel_graph0,{4825-4828},4828,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4822,5776,0x12c188010400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4817,3_Default/network-LlamaForCausalLM/model-LlamaModel/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/Cast-op0,kernel_graph0,{4834-4963-5774},5774,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4825,5253,0x12c180010200,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4817,4_Default/network-LlamaForCausalLM/Cast-op1,Default/network-LlamaForCausalLM/Cast-op1,kernel_graph0,{4826-4845-5206-5250},5250,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4828,4859,0x12c180018400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4817,6_Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Cast-op0,kernel_graph0,{4853-4857},4857,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4831,5702,0x12c18001c600,0,DefaultEnhancedAscendMemoryPool,524800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4817,7_Default/network-LlamaForCausalLM/model-LlamaModel/freqs_mgr-FreqsMgr/StridedSlice-op0,Default/network-LlamaForCausalLM/model-LlamaModel/freqs_mgr-FreqsMgr/StridedSlice-op0,kernel_graph0,{4832-4896-4919-5049-5064-5437-5458-5679-5700},5700,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4834,4840,0x12c18c010600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4817,9_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op0,kernel_graph0,{4838},4838,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4836,4844,0x12c194010800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4818,10_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{4842},4842,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4837,5783,0x12c18009c800,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4818,10_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{5779},5779,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4838,4841,0x12c1800a4a00,0,DefaultEnhancedAscendMemoryPool,16777728,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4818,10_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4842,5754,0x12c1800a4a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4819,11_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op1,kernel_graph0,{4867-4872-4887-4910-5724-5738-5752},5752,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4845,4853,0x12c180008800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4819,12_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/ReduceSum-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/ReduceSum-op0,kernel_graph0,{4851},4851,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4847,4849,0x12c180008a00,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,4847,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4848,4850,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,4848,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4851,5264,0x12c180008a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4819,13_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Add-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Add-op0,kernel_graph0,{5194-5261},5261,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4855,4860,0x12c1840a4c00,0,DefaultEnhancedAscendMemoryPool,2097664,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4820,15_Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/ExpandDims-op0,Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/ExpandDims-op0,kernel_graph0,{4857},4857,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4857,4863,0x12c1842a4e00,0,DefaultEnhancedAscendMemoryPool,16777728,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4821,16_Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Mul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Mul-op0,kernel_graph0,{4861},4861,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4861,4867,0x12c1852a5000,0,DefaultEnhancedAscendMemoryPool,16777728,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4821,17_Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Sub-op0,Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Sub-op0,kernel_graph0,{4863-4865},4865,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4865,5660,0x12c1840a4c00,0,DefaultEnhancedAscendMemoryPool,8389120,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4822,19_Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Cast-op1,kernel_graph0,{4948-5095-5413-5654},5654,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4869,5748,0x12c1848a4e00,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4823,21_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/Cast-op0,kernel_graph0,{4872-5745},5745,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4871,4879,0x12c18c010600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4825,22_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,kernel_graph0,{4874-4877},4877,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4872,4874,0x12c1868a5000,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4825,22_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4876,4883,0x12c190010800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4826,24_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,kernel_graph0,{4881},4881,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4877,4880,0x12c180008c00,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4826,24_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4881,4900,0x12c194010a00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4827,25_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op2,kernel_graph0,{4896-4898},4898,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4884,5763,0x12c18c010600,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4828,26_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/Cast-op0,kernel_graph0,{4887-5760},5760,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4886,4894,0x12c18e010800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4829,27_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,kernel_graph0,{4889-4892},4892,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4887,4889,0x12c1868a5000,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4829,27_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4891,4903,0x12c19c010c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4831,29_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,kernel_graph0,{4901},4901,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4892,4895,0x12c180008c00,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4831,29_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4896,4939,0x12c1a0010e00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4832,30_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op2,kernel_graph0,{4937},4937,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4898,4936,0x12c1a8011000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4832,31_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op1,kernel_graph0,{4934},4934,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4901,4923,0x12c18e010800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4832,32_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op0,kernel_graph0,{4919-4921},4921,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4904,5705,0x12c1868a5000,0,DefaultEnhancedAscendMemoryPool,524800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4832,33_Default/network-LlamaForCausalLM/model-LlamaModel/freqs_mgr-FreqsMgr/StridedSlice-op1,Default/network-LlamaForCausalLM/model-LlamaModel/freqs_mgr-FreqsMgr/StridedSlice-op1,kernel_graph0,{4905-4924-4934-5054-5069-5439-5460-5681-5703},5703,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4907,5734,0x12c196010a00,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4833,35_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/Cast-op0,kernel_graph0,{4910-5731},5731,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4909,4917,0x12c198010c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4834,36_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,kernel_graph0,{4912-4915},4915,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4910,4912,0x12c186925200,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4834,36_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4914,5658,0x12c1b0011200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4835,38_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,kernel_graph0,{4948-5654},5654,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4915,4918,0x12c180008c00,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4835,38_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4919,4929,0x12c198010c00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4838,39_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op0,kernel_graph0,{4927},4927,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4921,4926,0x12c1b4011400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4839,40_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op0,kernel_graph0,{4924},4924,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4924,4930,0x12c18e010800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4839,41_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op1,kernel_graph0,{4927},4927,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4927,4933,0x12c1b4011400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4841,42_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op0,kernel_graph0,{4931},4931,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4931,5656,0x12c18e010800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4842,43_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op1,kernel_graph0,{4948-5654},5654,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4934,4940,0x12c198010c00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4843,44_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op3,kernel_graph0,{4937},4937,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4937,4943,0x12c1a8011000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4844,45_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op1,kernel_graph0,{4941},4941,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4941,5657,0x12c198010c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4845,46_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op3,kernel_graph0,{4948-5654},5654,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4944,5661,0x12c186925200,0,DefaultEnhancedAscendMemoryPool,8389120,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4846,47_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{5654},5654,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4945,5662,0x12c187125400,0,DefaultEnhancedAscendMemoryPool,8389120,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4846,47_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{5654},5654,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4946,5663,0x12c180008800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4846,47_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{5654},5654,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4947,5664,0x12c19c010e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4846,47_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{4952-5654},5654,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4948,4950,0x12c1a0011000,0,DefaultEnhancedAscendMemoryPool,84936192,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4846,47_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4951,5634,0x12c1a0011000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4846,48_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,kernel_graph0,{4954-4959-5632},5632,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4952,4954,0x12c180008c00,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4846,48_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4956,5642,0x12c192010a00,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4847,50_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/Cast-op0,kernel_graph0,{4959-5640},5640,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4958,4965,0x12c1a4011200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4848,51_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,kernel_graph0,{4961-4963},4963,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4959,4961,0x12c194010c00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4848,51_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4963,5614,0x12c1a8011400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4848,53_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/Add-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/Add-op0,kernel_graph0,{4966-5004-5612},5612,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4966,4972,0x12c1b4011400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4851,54_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op0,kernel_graph0,{4970},4970,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4968,4976,0x12c1bc011600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4851,55_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{4974},4974,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4969,5621,0x12c187925600,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4851,55_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{5617},5617,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4970,4973,0x12c194010c00,0,DefaultEnhancedAscendMemoryPool,16777728,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4851,55_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4974,5593,0x12c1a4011200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4852,56_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op1,kernel_graph0,{4976-4981-4988-5577-5591},5591,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4978,5587,0x12c1b4011400,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4854,58_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/Cast-op0,kernel_graph0,{4981-5584},5584,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4980,5569,0x12c1b9611600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4855,59_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,kernel_graph0,{4983-4994-5567},5567,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4981,4983,0x12c194010c00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4855,59_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4985,5602,0x12c1c4211800,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4856,61_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/Cast-op0,kernel_graph0,{4988-5599},5599,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4987,5575,0x12c1c9811a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4857,62_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,kernel_graph0,{4991-5572},5572,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4988,4990,0x12c194010c00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4857,62_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4991,5565,0x12c1d4411c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4858,63_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/SiLU-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/SiLU-op0,kernel_graph0,{4992-4994-5563},5563,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4994,5552,0x12c1df011e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4861,65_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/Mul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/Mul-op0,kernel_graph0,{4995-5000-5550},5550,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4997,5560,0x12c1e9c12000,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4863,67_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/Cast-op0,kernel_graph0,{5000-5558},5558,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 4999,5006,0x12c1ef212200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4864,68_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,kernel_graph0,{5002-5004},5004,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5000,5002,0x12c194010c00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4864,68_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5004,5532,0x12c1f3212400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4866,70_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/Add-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/Add-op1,kernel_graph0,{5007-5110-5530},5530,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5007,5013,0x12c1f7212600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4866,71_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op0,kernel_graph0,{5011},5011,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5009,5017,0x12c1ff212800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4867,72_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{5015},5015,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5010,5539,0x12c18792d800,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4867,72_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{5535},5535,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5011,5014,0x12c194010c00,0,DefaultEnhancedAscendMemoryPool,16777728,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4867,72_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5015,5510,0x12c1ef212200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4868,73_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op1,kernel_graph0,{5017-5022-5037-5082-5480-5494-5508},5508,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5019,5504,0x12c1ac011600,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4870,75_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/Cast-op0,kernel_graph0,{5022-5501},5501,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5021,5029,0x12c1f7212600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4870,76_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,kernel_graph0,{5024-5027},5027,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5022,5024,0x12c1ae011800,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4870,76_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5026,5033,0x12c1fb212800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4873,78_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,kernel_graph0,{5031},5031,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5027,5030,0x12c180008c00,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4873,78_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5031,5053,0x12c1ff212a00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4873,79_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op2,kernel_graph0,{5049-5051},5051,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5034,5519,0x12c1f7212600,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4874,80_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/Cast-op0,kernel_graph0,{5037-5516},5516,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5036,5044,0x12c1f9212800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4875,81_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,kernel_graph0,{5039-5042},5042,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5037,5039,0x12c1ae011800,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4875,81_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5041,5048,0x12c207212c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4878,83_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,kernel_graph0,{5046},5046,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5042,5045,0x12c180008c00,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4878,83_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5046,5068,0x12c20b212e00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4878,84_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op0,kernel_graph0,{5064-5066},5066,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5049,5059,0x12c213213000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4881,85_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op2,kernel_graph0,{5057},5057,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5051,5056,0x12c21b213200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4882,86_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op1,kernel_graph0,{5054},5054,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5054,5060,0x12c1f9212800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4882,87_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op3,kernel_graph0,{5057},5057,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5057,5063,0x12c201212a00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4884,88_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op1,kernel_graph0,{5061},5061,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5061,5416,0x12c1f9212800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4884,89_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op3,kernel_graph0,{5095-5413},5413,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5064,5074,0x12c1fd212a00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4885,90_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op0,kernel_graph0,{5072},5072,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5066,5071,0x12c213213000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4885,91_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op0,kernel_graph0,{5069},5069,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5069,5075,0x12c205212c00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4887,92_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op1,kernel_graph0,{5072},5072,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5072,5078,0x12c20d212e00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4888,93_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op0,kernel_graph0,{5076},5076,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5076,5415,0x12c1fd212a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4888,94_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op1,kernel_graph0,{5095-5413},5413,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5079,5490,0x12c201212c00,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4889,95_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/Cast-op0,kernel_graph0,{5082-5487},5487,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5081,5089,0x12c203212e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4890,96_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,kernel_graph0,{5084-5087},5087,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5082,5084,0x12c1ae011800,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4890,96_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5086,5417,0x12c207213000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4893,98_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,kernel_graph0,{5095-5413},5413,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5087,5090,0x12c180008c00,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4893,98_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5091,5419,0x12c1ae011800,0,DefaultEnhancedAscendMemoryPool,8389120,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4893,99_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{5413},5413,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5092,5420,0x12c1ae811a00,0,DefaultEnhancedAscendMemoryPool,8389120,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4893,99_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{5413},5413,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5093,5421,0x12c180008c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4893,99_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{5413},5413,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5094,5422,0x12c203212e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4893,99_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{5099-5413},5413,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5095,5097,0x12c20b213200,0,DefaultEnhancedAscendMemoryPool,84936192,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4893,99_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5098,5393,0x12c20b213200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4896,100_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,kernel_graph0,{5101-5106-5391},5391,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5099,5101,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4896,100_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5103,5401,0x12c20f213400,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4897,102_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/Cast-op0,kernel_graph0,{5106-5399},5399,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5105,5112,0x12c211213600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4898,103_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,kernel_graph0,{5108-5110},5110,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5106,5108,0x12c194010c00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4898,103_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5110,5368,0x12c215213800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4899,105_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/Add-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/Add-op0,kernel_graph0,{5113-5151-5366},5366,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5113,5119,0x12c219213a00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4901,106_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op0,kernel_graph0,{5117},5117,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5115,5123,0x12c221213c00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4902,107_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{5121},5121,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5116,5375,0x12c187935a00,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4902,107_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{5371},5371,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5117,5120,0x12c194010c00,0,DefaultEnhancedAscendMemoryPool,16777728,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4902,107_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5121,5347,0x12c211213600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4903,108_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op1,kernel_graph0,{5123-5128-5135-5331-5345},5345,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5125,5341,0x12c219213a00,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4905,110_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/Cast-op0,kernel_graph0,{5128-5338},5338,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5127,5323,0x12c21e813c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4905,111_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,kernel_graph0,{5130-5141-5321},5321,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5128,5130,0x12c194010c00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4905,111_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5132,5356,0x12c229413e00,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4905,113_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/Cast-op0,kernel_graph0,{5135-5353},5353,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5134,5329,0x12c22ea14000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4906,114_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,kernel_graph0,{5138-5326},5326,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5135,5137,0x12c194010c00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4906,114_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5138,5319,0x12c239614200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4907,115_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/SiLU-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/SiLU-op0,kernel_graph0,{5139-5141-5317},5317,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5141,5306,0x12c244214400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4908,117_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/Mul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/Mul-op0,kernel_graph0,{5142-5147-5304},5304,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5144,5314,0x12c24ee14600,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4911,119_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/Cast-op0,kernel_graph0,{5147-5312},5312,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5146,5153,0x12c254414800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4911,120_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,kernel_graph0,{5149-5151},5151,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5147,5149,0x12c194010c00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4911,120_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5151,5285,0x12c258414a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4913,122_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/Add-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/Add-op1,kernel_graph0,{5154-5283},5283,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5154,5160,0x12c25c414c00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4915,123_Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Cast-op0,kernel_graph0,{5158},5158,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5156,5164,0x12c264414e00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4915,124_Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{5162},5162,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5157,5292,0x12c18793dc00,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4915,124_Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{5288},5288,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5158,5161,0x12c194010c00,0,DefaultEnhancedAscendMemoryPool,16777728,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4915,124_Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5162,5268,0x12c254414800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4916,125_Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Cast-op1,kernel_graph0,{5164-5169-5266},5266,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5166,5277,0x12c25c414c00,0,DefaultEnhancedAscendMemoryPool,262144512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4919,127_Default/network-LlamaForCausalLM/lm_head-Linear/Cast-op0,Default/network-LlamaForCausalLM/lm_head-Linear/Cast-op0,kernel_graph0,{5169-5274},5274,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5168,5174,0x12c26be14e00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4920,128_Default/network-LlamaForCausalLM/lm_head-Linear/MatMul-op0,Default/network-LlamaForCausalLM/lm_head-Linear/MatMul-op0,kernel_graph0,{5172},5172,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5169,5171,0x12c194010c00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4920,128_Default/network-LlamaForCausalLM/lm_head-Linear/MatMul-op0,Default/network-LlamaForCausalLM/lm_head-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5172,5181,0x12c28b215000,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4920,129_Default/network-LlamaForCausalLM/Cast-op0,Default/network-LlamaForCausalLM/Cast-op0,kernel_graph0,{5176-5179},5179,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5175,5182,0x12c187945e00,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4921,130_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/ReduceMax-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/ReduceMax-op0,kernel_graph0,{5179},5179,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5176,5178,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4921,130_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/ReduceMax-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/ReduceMax-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5179,5215,0x12c2c9a15200,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4922,131_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Sub-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Sub-op0,kernel_graph0,{5185-5213},5213,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5183,5196,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4922,132_Default/Mul-op0,Default/Mul-op0,kernel_graph0,{5194},5194,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5185,5193,0x12c26be14e00,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4924,133_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Exp-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Exp-op0,kernel_graph0,{5187},5187,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5187,5212,0x12c187945e00,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4925,134_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/ReduceSum-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/ReduceSum-op0,kernel_graph0,{5210},5210,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5189,5191,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,5189,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5190,5192,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,5190,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5194,5200,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4925,135_Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_RealDiv/RealDiv-op0,Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_RealDiv/RealDiv-op0,kernel_graph0,{5198},5198,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5197,5205,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4927,136_Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_RealDiv/SumExt-op0,Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_RealDiv/SumExt-op0,kernel_graph0,{5201-5203},5203,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5198,5201,0x12c180009200,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4927,136_Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_RealDiv/SumExt-op0,Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_RealDiv/SumExt-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5203,5208,0x12c18794e000,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4928,138_Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_ReduceSum/Tile-op0,Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_ReduceSum/Tile-op0,kernel_graph0,{5206},5206,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5206,5232,0x12c187956200,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4931,139_Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_Mul/Mul-op0,Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_Mul/Mul-op0,kernel_graph0,{5208-5229},5229,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5210,5216,0x12c18794e000,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4932,141_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Log-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Log-op0,kernel_graph0,{5213},5213,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5213,5238,0x12c26be14e00,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4934,142_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Sub-op1,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Sub-op1,kernel_graph0,{5224-5236},5236,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5217,5223,0x12c187945e00,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4935,143_Default/network-LlamaForCausalLM/StridedSlice-op1,Default/network-LlamaForCausalLM/StridedSlice-op1,kernel_graph0,{5219-5221},5221,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5221,5239,0x12c2aa615000,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4937,145_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/OneHot-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/OneHot-op0,kernel_graph0,{5226-5236},5236,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5224,5228,0x12c2e8e15200,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4938,146_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Exp-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Exp-op0,kernel_graph0,{5226},5226,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5226,5231,0x12c327615400,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4940,147_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Sub-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Sub-op0,kernel_graph0,{5229},5229,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5229,5235,0x12c2e8e15200,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4941,148_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Mul-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Mul-op0,kernel_graph0,{5233},5233,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5233,5276,0x12c327615400,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4942,149_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op0,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op0,kernel_graph0,{5266-5274},5274,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5236,5242,0x12c2e8e15200,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4942,150_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Mul-op1,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Mul-op1,kernel_graph0,{5240},5240,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5240,5249,0x12c26be14e00,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4948,151_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Neg-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Neg-op0,kernel_graph0,{5243},5243,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5243,5252,0x12c187945e00,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4948,152_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/ReduceSum-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/ReduceSum-op0,kernel_graph0,{5250},5250,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5245,5247,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,5245,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5246,5248,0x12c180009200,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,5246,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5250,5260,0x12c18794e000,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4949,153_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Mul-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Mul-op0,kernel_graph0,{5254},5254,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5254,5263,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4949,154_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/ReduceSum-op1,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/ReduceSum-op1,kernel_graph0,{5261},5261,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5256,5258,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,5256,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5257,5259,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,5257,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5261,7097,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4949,155_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/RealDiv-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/RealDiv-op0,kernel_graph0,{7095},7095,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5265,5272,0x12c346a15600,0,DefaultEnhancedAscendMemoryPool,262144512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4952,156_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op29,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op29,kernel_graph0,{5270},5270,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5266,5269,0x12c194010c00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4952,156_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op29,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op29,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5270,5851,0x12c26be14e00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4953,157_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op29,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op29,kernel_graph0,{5849},5849,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5273,5282,0x12c254414800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4953,158_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op0,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op0,kernel_graph0,{5278-5280},5280,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5274,5278,0x12c346a15600,0,DefaultEnhancedAscendMemoryPool,155190272,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4953,158_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op0,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5280,5290,0x12c25c414c00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4955,160_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op1,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op1,kernel_graph0,{5288},5288,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5283,5291,0x12c28b215000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4956,161_Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Cast-op2,kernel_graph0,{5288},5288,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5286,5296,0x12c254414800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4957,162_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op0,kernel_graph0,{5294},5294,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5287,5866,0x12c18000a400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4957,162_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op0,kernel_graph0,{5864},5864,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5288,5293,0x12c194010c00,0,DefaultEnhancedAscendMemoryPool,16778752,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4957,162_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5294,5383,0x12c25c414a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4957,163_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op0,kernel_graph0,{5298-5304-5312-5380},5380,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5297,5302,0x12c180008a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4959,164_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Mul-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Mul-op0,kernel_graph0,{5300},5300,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5300,6066,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4960,166_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Maximum-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Maximum-op0,kernel_graph0,{6064},6064,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5303,5310,0x12c254414800,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4961,167_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op27,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op27,kernel_graph0,{5308},5308,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5304,5307,0x12c194010c00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4961,167_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op27,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op27,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5308,5845,0x12c244214400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4962,168_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op27,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op27,kernel_graph0,{5843},5843,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5311,5324,0x12c260414c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4963,169_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op1,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op1,kernel_graph0,{5315-5317-5321},5321,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5312,5315,0x12c194010c00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4963,169_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op1,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5317,5340,0x12c24ee14600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4964,171_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op0,kernel_graph0,{5319-5331-5338},5338,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5321,5328,0x12c239614200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4967,173_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op1,kernel_graph0,{5324-5326},5326,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5326,5355,0x12c21e813c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4970,175_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/Grad_SiLU/SiLUGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/Grad_SiLU/SiLUGrad-op0,kernel_graph0,{5345-5353},5353,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5330,5336,0x12c260414c00,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4971,176_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op28,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op28,kernel_graph0,{5334},5334,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5331,5333,0x12c194010c00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4971,176_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op28,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op28,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5334,5848,0x12c22ea14000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4971,177_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op28,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op28,kernel_graph0,{5846},5846,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5337,5361,0x12c239614200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4974,178_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op2,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op2,kernel_graph0,{5342-5359},5359,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5338,5342,0x12c194010c00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4974,178_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op2,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5344,5351,0x12c219213a00,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4975,180_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op26,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op26,kernel_graph0,{5349},5349,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5345,5348,0x12c194010c00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4975,180_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op26,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op26,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5349,5842,0x12c260414c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4976,181_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op26,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op26,kernel_graph0,{5840},5840,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5352,5362,0x12c211213600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4977,182_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op3,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op3,kernel_graph0,{5357-5359},5359,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5353,5357,0x12c194010c00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4977,182_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op3,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op3,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5359,5365,0x12c23d614400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4979,184_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/AddN-op0,kernel_graph0,{5363},5363,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5363,5373,0x12c24ee14600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4979,185_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op2,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op2,kernel_graph0,{5371},5371,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5366,5374,0x12c239614200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4981,186_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op2,kernel_graph0,{5371},5371,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5369,5379,0x12c211213600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4982,187_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op1,kernel_graph0,{5377},5377,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5370,5860,0x12c180010200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4982,187_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op1,kernel_graph0,{5858},5858,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5371,5376,0x12c194010c00,0,DefaultEnhancedAscendMemoryPool,16778752,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4982,187_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5377,5382,0x12c239614200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4982,188_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op1,kernel_graph0,{5380},5380,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5380,5547,0x12c23d614400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4983,189_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/AddN-op0,kernel_graph0,{5385-5391-5399-5544},5544,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5384,5389,0x12c180008a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4984,190_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Sub-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Sub-op0,kernel_graph0,{5387},5387,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5387,6089,0x12c180009200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4986,192_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/LessEqual-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/LessEqual-op0,kernel_graph0,{6069-6087},6087,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5390,5397,0x12c241614600,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4986,193_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op25,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op25,kernel_graph0,{5395},5395,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5391,5394,0x12c194010c00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4986,193_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op25,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op25,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5395,5839,0x12c20b213200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4988,194_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op25,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op25,kernel_graph0,{5837},5837,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5398,5407,0x12c239614200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4989,195_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op4,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op4,kernel_graph0,{5402-5405},5405,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5399,5402,0x12c194010c00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4989,195_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op4,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op4,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5404,5418,0x12c24ee14600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4991,197_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op0,kernel_graph0,{5413},5413,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5405,5408,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4991,197_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5409,5430,0x12c239614200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4992,198_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,kernel_graph0,{5428},5428,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5410,5427,0x12c252e14800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4992,198_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,kernel_graph0,{5425},5425,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5411,5434,0x12c256e14a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4992,198_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,kernel_graph0,{5432},5432,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5412,5423,0x12c180008a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4992,198_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5413,5424,0x12c20f213400,0,DefaultEnhancedAscendMemoryPool,515969024,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4992,198_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5425,5441,0x12c1f9212800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4992,199_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op5,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op5,kernel_graph0,{5437-5439},5439,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5428,5462,0x12c203212e00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4992,200_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op3,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op3,kernel_graph0,{5458-5460},5460,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5431,5489,0x12c239614200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4993,201_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op3,kernel_graph0,{5435-5480-5487},5487,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5432,5435,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4993,201_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op3,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5437,5447,0x12c24ee14600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4995,203_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op4,kernel_graph0,{5445},5445,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5439,5444,0x12c256e14800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4995,204_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op5,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op5,kernel_graph0,{5442},5442,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5442,5448,0x12c1f9212800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4996,205_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op1,kernel_graph0,{5445},5445,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5445,5451,0x12c256e14800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4997,206_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op1,kernel_graph0,{5449},5449,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5449,5455,0x12c24ee14600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4998,207_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op6,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op6,kernel_graph0,{5453},5453,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5452,5503,0x12c1f9212800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4998,208_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op2,kernel_graph0,{5456-5494-5501},5501,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5453,5456,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,4998,208_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5458,5468,0x12c24ee14600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5001,210_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op2,kernel_graph0,{5466},5466,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5460,5465,0x12c256e14800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5001,211_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op3,kernel_graph0,{5463},5463,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5463,5469,0x12c203212e00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5002,212_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op0,kernel_graph0,{5466},5466,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5466,5472,0x12c256e14800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5003,213_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op0,kernel_graph0,{5470},5470,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5470,5476,0x12c1fd212a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5004,214_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op4,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op4,kernel_graph0,{5474},5474,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5473,5518,0x12c203212e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5005,215_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op1,kernel_graph0,{5477-5508-5516},5516,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5474,5477,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5005,215_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5479,5485,0x12c241614600,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5007,217_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op24,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op24,kernel_graph0,{5483},5483,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5480,5482,0x12c1ae011800,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5007,217_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op24,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op24,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5483,5836,0x12c1fd212a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5008,218_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op24,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op24,kernel_graph0,{5834},5834,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5486,5526,0x12c207213000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5009,219_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op7,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op7,kernel_graph0,{5491-5522},5522,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5487,5491,0x12c1ae011800,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5009,219_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op7,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op7,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5493,5499,0x12c201212c00,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5012,221_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op23,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op23,kernel_graph0,{5497},5497,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5494,5496,0x12c1ae011800,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5012,221_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op23,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op23,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5497,5833,0x12c239614200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5012,222_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op23,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op23,kernel_graph0,{5831},5831,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5500,5525,0x12c24ee14600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5015,223_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op6,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op6,kernel_graph0,{5505-5522},5522,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5501,5505,0x12c1ae011800,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5015,223_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op6,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op6,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5507,5514,0x12c201212c00,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5016,225_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op22,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op22,kernel_graph0,{5512},5512,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5508,5511,0x12c194010c00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5016,225_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op22,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op22,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5512,5830,0x12c1ef212200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5017,226_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op22,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op22,kernel_graph0,{5828},5828,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5515,5524,0x12c1f9212800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5018,227_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op5,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op5,kernel_graph0,{5520-5522},5522,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5516,5520,0x12c194010c00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5018,227_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op5,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op5,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5522,5529,0x12c201212c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5020,229_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/AddN-op0,kernel_graph0,{5527},5527,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5527,5537,0x12c24ee14600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5022,230_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op7,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op7,kernel_graph0,{5535},5535,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5530,5538,0x12c256e14800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5022,231_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op2,kernel_graph0,{5535},5535,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5533,5543,0x12c1f3212400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5023,232_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op2,kernel_graph0,{5541},5541,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5534,5863,0x12c180014400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5023,232_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op2,kernel_graph0,{5861},5861,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5535,5540,0x12c25ee14a00,0,DefaultEnhancedAscendMemoryPool,16778752,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5023,232_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5541,5546,0x12c201212c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5024,233_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op2,kernel_graph0,{5544},5544,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5544,5629,0x12c205212e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5025,234_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/AddN-op0,kernel_graph0,{5547-5550-5558-5626},5626,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5549,5556,0x12c23d614400,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5028,236_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op20,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op20,kernel_graph0,{5554},5554,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5550,5553,0x12c242c14600,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5028,236_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op20,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op20,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5554,5824,0x12c1df011e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5028,237_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op20,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op20,kernel_graph0,{5822},5822,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5557,5570,0x12c24ee14600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5031,238_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op8,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op8,kernel_graph0,{5561-5563-5567},5567,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5558,5561,0x12c194010c00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5031,238_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op8,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op8,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5563,5586,0x12c20f213400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5032,240_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op6,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op6,kernel_graph0,{5565-5577-5584},5584,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5567,5574,0x12c1d4411c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5034,242_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op7,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op7,kernel_graph0,{5570-5572},5572,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5572,5601,0x12c1b9611600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5035,244_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/Grad_SiLU/SiLUGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/Grad_SiLU/SiLUGrad-op1,kernel_graph0,{5591-5599},5599,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5576,5582,0x12c1e9c12000,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5037,245_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op21,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op21,kernel_graph0,{5580},5580,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5577,5579,0x12c194010c00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5037,245_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op21,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op21,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5580,5827,0x12c24ee14600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5038,246_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op21,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op21,kernel_graph0,{5825},5825,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5583,5607,0x12c201212c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5039,247_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op9,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op9,kernel_graph0,{5588-5605},5605,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5584,5588,0x12c194010c00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5039,247_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op9,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op9,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5590,5597,0x12c1b4011400,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5040,249_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op19,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op19,kernel_graph0,{5595},5595,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5591,5594,0x12c194010c00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5040,249_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op19,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op19,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5595,5821,0x12c1c9811a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5042,250_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op19,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op19,kernel_graph0,{5819},5819,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5598,5608,0x12c1a4011200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5043,251_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op10,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op10,kernel_graph0,{5603-5605},5605,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5599,5603,0x12c194010c00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5043,251_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op10,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op10,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5605,5611,0x12c1e9c12000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5046,253_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/AddN-op0,kernel_graph0,{5609},5609,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5609,5619,0x12c1f3212400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5047,254_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op8,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op8,kernel_graph0,{5617},5617,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5612,5620,0x12c1d4411c00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5047,255_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op2,kernel_graph0,{5617},5617,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5615,5625,0x12c1a4011200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5049,256_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op3,kernel_graph0,{5623},5623,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5616,5854,0x12c18792d800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5049,256_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op3,kernel_graph0,{5852},5852,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5617,5622,0x12c194010c00,0,DefaultEnhancedAscendMemoryPool,16778752,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5049,256_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op3,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5623,5628,0x12c201212c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5050,257_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op3,kernel_graph0,{5626},5626,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5626,5791,0x12c1e9c12000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5050,258_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/AddN-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/AddN-op1,kernel_graph0,{5629-5632-5640-5788},5788,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5631,5638,0x12c259a14800,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5052,260_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op18,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op18,kernel_graph0,{5636},5636,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5632,5635,0x12c1edc12200,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5052,260_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op18,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op18,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5636,5818,0x12c25ba14a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5052,261_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op18,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op18,kernel_graph0,{5816},5816,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5639,5648,0x12c23d614400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5053,262_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op11,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op11,kernel_graph0,{5643-5646},5646,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5640,5643,0x12c1edc12200,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5053,262_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op11,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op11,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5645,5659,0x12c1f3212400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5055,264_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op4,kernel_graph0,{5654},5654,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5646,5649,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5055,264_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op4,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5650,5672,0x12c1f7212600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5057,265_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,kernel_graph0,{5670},5670,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5651,5669,0x12c23d614400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5057,265_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,kernel_graph0,{5667},5667,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5652,5676,0x12c201212c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5057,265_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,kernel_graph0,{5674},5674,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5653,5665,0x12c180008a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5057,265_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5654,5666,0x12c20f213400,0,DefaultEnhancedAscendMemoryPool,515969024,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5057,265_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5667,5683,0x12c18e010800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5058,266_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op11,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op11,kernel_graph0,{5679-5681},5681,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5670,5706,0x12c1d4411c00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5058,267_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op9,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op9,kernel_graph0,{5700-5703},5703,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5673,5733,0x12c205212e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5061,268_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op7,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op7,kernel_graph0,{5677-5724-5731},5731,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5674,5677,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5061,268_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op7,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op7,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5679,5689,0x12c1f3212400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5062,270_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op10,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op10,kernel_graph0,{5687},5687,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5681,5686,0x12c20f213400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5064,271_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op11,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op11,kernel_graph0,{5684},5684,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5684,5690,0x12c18e010800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5065,272_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op3,kernel_graph0,{5687},5687,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5687,5693,0x12c20f213400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5065,273_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op1,kernel_graph0,{5691},5691,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5691,5697,0x12c201212c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5066,274_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op12,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op12,kernel_graph0,{5695},5695,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5694,5747,0x12c23d614400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5067,275_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op6,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op6,kernel_graph0,{5698-5738-5745},5745,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5695,5698,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5067,275_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op6,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op6,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5700,5712,0x12c18e010800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5069,277_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op8,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op8,kernel_graph0,{5710},5710,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5703,5709,0x12c1f3212400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5070,278_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op9,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op9,kernel_graph0,{5707},5707,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5707,5713,0x12c1d4411c00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5071,279_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op2,kernel_graph0,{5710},5710,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5710,5716,0x12c1f3212400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5072,280_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op0,kernel_graph0,{5714},5714,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5714,5720,0x12c201212c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5073,281_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op10,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op10,kernel_graph0,{5718},5718,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5717,5762,0x12c18e010800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5073,282_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op5,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op5,kernel_graph0,{5721-5752-5760},5760,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5718,5721,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5073,282_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op5,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op5,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5723,5729,0x12c209213000,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5077,284_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op17,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op17,kernel_graph0,{5727},5727,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5724,5726,0x12c1edc12200,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5077,284_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op17,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op17,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5727,5815,0x12c201212c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5077,285_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op17,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op17,kernel_graph0,{5813},5813,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5730,5770,0x12c1f3212400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5079,286_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op14,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op14,kernel_graph0,{5735-5766},5766,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5731,5735,0x12c1edc12200,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5079,286_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op14,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op14,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5737,5743,0x12c259a14800,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5080,288_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op16,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op16,kernel_graph0,{5741},5741,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5738,5740,0x12c1edc12200,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5080,288_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op16,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op16,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5741,5812,0x12c1f7212600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5082,289_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op16,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op16,kernel_graph0,{5810},5810,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5744,5769,0x12c205212e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5083,290_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op13,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op13,kernel_graph0,{5749-5766},5766,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5745,5749,0x12c1edc12200,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5083,290_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op13,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op13,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5751,5758,0x12c1fb212800,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5084,292_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op15,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op15,kernel_graph0,{5756},5756,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5752,5755,0x12c1edc12200,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5084,292_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op15,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op15,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5756,5809,0x12c23d614400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5085,293_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op15,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op15,kernel_graph0,{5807},5807,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5759,5768,0x12c1800a4a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5087,294_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op12,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op12,kernel_graph0,{5764-5766},5766,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5760,5764,0x12c1edc12200,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5087,294_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op12,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op12,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5766,5773,0x12c1d4411c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5090,296_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/AddN-op0,kernel_graph0,{5771},5771,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5771,5781,0x12c20f213400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5095,297_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op13,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op13,kernel_graph0,{5779},5779,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5774,5782,0x12c1d4411c00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5096,298_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op2,kernel_graph0,{5779},5779,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5777,5787,0x12c217213600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5096,299_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op4,kernel_graph0,{5785},5785,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5778,5857,0x12c180018600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5096,299_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op4,kernel_graph0,{5855},5855,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5779,5784,0x12c1edc12200,0,DefaultEnhancedAscendMemoryPool,16778752,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5096,299_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op4,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5785,5790,0x12c1f3212400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5096,300_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op4,kernel_graph0,{5788},5788,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5788,5794,0x12c205212e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5098,301_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/AddN-op0,kernel_graph0,{5792},5792,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5792,5797,0x12c1d4411c00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5099,302_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op14,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op14,kernel_graph0,{5795},5795,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5795,5806,0x12c20f213400,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5100,303_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/tok_embeddings-LlamaEmbedding/Grad_Gather/UnsortedSegmentSum-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/tok_embeddings-LlamaEmbedding/Grad_Gather/UnsortedSegmentSum-op0,kernel_graph0,{5804},5804,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5799,5803,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5100,304_Default/Mul-op1,Default/Mul-op1,kernel_graph0,{5801},5801,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5801,5867,0x12c180009600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5101,305_Default/Reciprocal-op0,Default/Reciprocal-op0,kernel_graph0,{5804-5807-5810-5813-5816-5819-5822-5825-5828-5831-5834-5837-5840-5843-5846-5849-5852-5855-5858-5861-5864},5864,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5804,6102,0x12c187931a00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5102,306_Default/Mul-op2,Default/Mul-op2,kernel_graph0,{5868-6028-6100},6100,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5807,6105,0x12c1f3212400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5103,307_Default/Mul-op3,Default/Mul-op3,kernel_graph0,{5876-6028-6103},6103,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5810,6108,0x12c1e9c12000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5104,308_Default/Mul-op4,Default/Mul-op4,kernel_graph0,{5884-6028-6106},6106,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5813,6111,0x12c1f7212600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5104,309_Default/Mul-op5,Default/Mul-op5,kernel_graph0,{5892-6028-6109},6109,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5816,6114,0x12c23d614400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5106,310_Default/Mul-op6,Default/Mul-op6,kernel_graph0,{5900-6028-6112},6112,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5819,6117,0x12c1d4411c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5107,311_Default/Mul-op7,Default/Mul-op7,kernel_graph0,{5908-6028-6115},6115,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5822,6120,0x12c20f213400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5107,312_Default/Mul-op8,Default/Mul-op8,kernel_graph0,{5916-6028-6118},6118,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5825,6123,0x12c1df011e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5108,313_Default/Mul-op9,Default/Mul-op9,kernel_graph0,{5924-6028-6121},6121,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5828,6126,0x12c18001c800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5109,314_Default/Mul-op10,Default/Mul-op10,kernel_graph0,{5932-6028-6124},6124,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5831,6129,0x12c1edc12200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5110,315_Default/Mul-op11,Default/Mul-op11,kernel_graph0,{5940-6028-6127},6127,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5834,6132,0x12c239614200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5111,316_Default/Mul-op12,Default/Mul-op12,kernel_graph0,{5948-6028-6130},6130,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5837,6135,0x12c1fb212800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5111,317_Default/Mul-op13,Default/Mul-op13,kernel_graph0,{5956-6028-6133},6133,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5840,6138,0x12c1ff212a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5113,318_Default/Mul-op14,Default/Mul-op14,kernel_graph0,{5964-6028-6136},6136,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5843,6141,0x12c219e13600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5114,319_Default/Mul-op15,Default/Mul-op15,kernel_graph0,{5972-6028-6139},6139,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5846,6144,0x12c241614600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5114,320_Default/Mul-op16,Default/Mul-op16,kernel_graph0,{5980-6028-6142},6142,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5849,6147,0x12c24c214800,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5117,321_Default/Mul-op17,Default/Mul-op17,kernel_graph0,{5988-6028-6145},6145,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5852,6150,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5118,322_Default/Mul-op18,Default/Mul-op18,kernel_graph0,{5996-6028-6148},6148,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5855,6153,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5118,323_Default/Mul-op19,Default/Mul-op19,kernel_graph0,{6004-6028-6151},6151,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5858,6156,0x12c180018600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5121,324_Default/Mul-op20,Default/Mul-op20,kernel_graph0,{6012-6028-6154},6154,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5861,6159,0x12c180010200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5122,325_Default/Mul-op21,Default/Mul-op21,kernel_graph0,{6020-6028-6157},6157,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5864,6162,0x12c180014400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5122,326_Default/Mul-op22,Default/Mul-op22,kernel_graph0,{6028-6030-6160},6160,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5868,5873,0x12c1a6d31c00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5123,327_Default/clip_grad_norm-ClipGradNorm/Square-op0,Default/clip_grad_norm-ClipGradNorm/Square-op0,kernel_graph0,{5871},5871,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5870,6040,0x12c180008a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5124,328_Default/clip_grad_norm-ClipGradNorm/SumExt-op0,Default/clip_grad_norm-ClipGradNorm/SumExt-op0,kernel_graph0,{5874-6038},6038,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5871,5874,0x12c18000a400,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5124,328_Default/clip_grad_norm-ClipGradNorm/SumExt-op0,Default/clip_grad_norm-ClipGradNorm/SumExt-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5876,5881,0x12c209e12c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5126,330_Default/clip_grad_norm-ClipGradNorm/Square-op1,Default/clip_grad_norm-ClipGradNorm/Square-op1,kernel_graph0,{5879},5879,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5878,6041,0x12c180008c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5127,331_Default/clip_grad_norm-ClipGradNorm/SumExt-op1,Default/clip_grad_norm-ClipGradNorm/SumExt-op1,kernel_graph0,{5882-6038},6038,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5879,5882,0x12c18000a400,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5127,331_Default/clip_grad_norm-ClipGradNorm/SumExt-op1,Default/clip_grad_norm-ClipGradNorm/SumExt-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5884,5889,0x12c209e12c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5129,333_Default/clip_grad_norm-ClipGradNorm/Square-op2,Default/clip_grad_norm-ClipGradNorm/Square-op2,kernel_graph0,{5887},5887,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5886,6042,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5129,334_Default/clip_grad_norm-ClipGradNorm/SumExt-op2,Default/clip_grad_norm-ClipGradNorm/SumExt-op2,kernel_graph0,{5890-6038},6038,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5887,5890,0x12c18000a400,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5129,334_Default/clip_grad_norm-ClipGradNorm/SumExt-op2,Default/clip_grad_norm-ClipGradNorm/SumExt-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5892,5897,0x12c209e12c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5131,336_Default/clip_grad_norm-ClipGradNorm/Square-op3,Default/clip_grad_norm-ClipGradNorm/Square-op3,kernel_graph0,{5895},5895,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5894,6043,0x12c180009600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5132,337_Default/clip_grad_norm-ClipGradNorm/SumExt-op3,Default/clip_grad_norm-ClipGradNorm/SumExt-op3,kernel_graph0,{5898-6038},6038,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5895,5898,0x12c18000a400,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5132,337_Default/clip_grad_norm-ClipGradNorm/SumExt-op3,Default/clip_grad_norm-ClipGradNorm/SumExt-op3,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5900,5905,0x12c209e12c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5133,339_Default/clip_grad_norm-ClipGradNorm/Square-op4,Default/clip_grad_norm-ClipGradNorm/Square-op4,kernel_graph0,{5903},5903,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5902,6044,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5135,340_Default/clip_grad_norm-ClipGradNorm/SumExt-op4,Default/clip_grad_norm-ClipGradNorm/SumExt-op4,kernel_graph0,{5906-6038},6038,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5903,5906,0x12c18000a400,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5135,340_Default/clip_grad_norm-ClipGradNorm/SumExt-op4,Default/clip_grad_norm-ClipGradNorm/SumExt-op4,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5908,5913,0x12c224a13800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5136,342_Default/clip_grad_norm-ClipGradNorm/Square-op5,Default/clip_grad_norm-ClipGradNorm/Square-op5,kernel_graph0,{5911},5911,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5910,6045,0x12c180009a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5138,343_Default/clip_grad_norm-ClipGradNorm/SumExt-op5,Default/clip_grad_norm-ClipGradNorm/SumExt-op5,kernel_graph0,{5914-6038},6038,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5911,5914,0x12c18000a400,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5138,343_Default/clip_grad_norm-ClipGradNorm/SumExt-op5,Default/clip_grad_norm-ClipGradNorm/SumExt-op5,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5916,5921,0x12c224a13800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5139,345_Default/clip_grad_norm-ClipGradNorm/Square-op6,Default/clip_grad_norm-ClipGradNorm/Square-op6,kernel_graph0,{5919},5919,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5918,6046,0x12c180009c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5139,346_Default/clip_grad_norm-ClipGradNorm/SumExt-op6,Default/clip_grad_norm-ClipGradNorm/SumExt-op6,kernel_graph0,{5922-6038},6038,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5919,5922,0x12c18000a400,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5139,346_Default/clip_grad_norm-ClipGradNorm/SumExt-op6,Default/clip_grad_norm-ClipGradNorm/SumExt-op6,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5924,5929,0x12c224a13800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5141,348_Default/clip_grad_norm-ClipGradNorm/Square-op7,Default/clip_grad_norm-ClipGradNorm/Square-op7,kernel_graph0,{5927},5927,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5926,6047,0x12c180009e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5142,349_Default/clip_grad_norm-ClipGradNorm/SumExt-op7,Default/clip_grad_norm-ClipGradNorm/SumExt-op7,kernel_graph0,{5930-6038},6038,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5927,5930,0x12c18000a400,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5142,349_Default/clip_grad_norm-ClipGradNorm/SumExt-op7,Default/clip_grad_norm-ClipGradNorm/SumExt-op7,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5932,5937,0x12c209e12c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5143,351_Default/clip_grad_norm-ClipGradNorm/Square-op8,Default/clip_grad_norm-ClipGradNorm/Square-op8,kernel_graph0,{5935},5935,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5934,6048,0x12c18000a400,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5144,352_Default/clip_grad_norm-ClipGradNorm/SumExt-op8,Default/clip_grad_norm-ClipGradNorm/SumExt-op8,kernel_graph0,{5938-6038},6038,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5935,5938,0x12c18000a600,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5144,352_Default/clip_grad_norm-ClipGradNorm/SumExt-op8,Default/clip_grad_norm-ClipGradNorm/SumExt-op8,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5940,5945,0x12c209e12c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5145,354_Default/clip_grad_norm-ClipGradNorm/Square-op9,Default/clip_grad_norm-ClipGradNorm/Square-op9,kernel_graph0,{5943},5943,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5942,6049,0x12c18000a600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5147,355_Default/clip_grad_norm-ClipGradNorm/SumExt-op9,Default/clip_grad_norm-ClipGradNorm/SumExt-op9,kernel_graph0,{5946-6038},6038,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5943,5946,0x12c18000a800,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5147,355_Default/clip_grad_norm-ClipGradNorm/SumExt-op9,Default/clip_grad_norm-ClipGradNorm/SumExt-op9,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5948,5953,0x12c209e12c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5148,357_Default/clip_grad_norm-ClipGradNorm/Square-op10,Default/clip_grad_norm-ClipGradNorm/Square-op10,kernel_graph0,{5951},5951,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5950,6050,0x12c18000a800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5149,358_Default/clip_grad_norm-ClipGradNorm/SumExt-op10,Default/clip_grad_norm-ClipGradNorm/SumExt-op10,kernel_graph0,{5954-6038},6038,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5951,5954,0x12c18000aa00,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5149,358_Default/clip_grad_norm-ClipGradNorm/SumExt-op10,Default/clip_grad_norm-ClipGradNorm/SumExt-op10,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5956,5961,0x12c209e12c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5151,360_Default/clip_grad_norm-ClipGradNorm/Square-op11,Default/clip_grad_norm-ClipGradNorm/Square-op11,kernel_graph0,{5959},5959,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5958,6051,0x12c18000aa00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5152,361_Default/clip_grad_norm-ClipGradNorm/SumExt-op11,Default/clip_grad_norm-ClipGradNorm/SumExt-op11,kernel_graph0,{5962-6038},6038,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5959,5962,0x12c18000ac00,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5152,361_Default/clip_grad_norm-ClipGradNorm/SumExt-op11,Default/clip_grad_norm-ClipGradNorm/SumExt-op11,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5964,5969,0x12c224a13800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5154,363_Default/clip_grad_norm-ClipGradNorm/Square-op12,Default/clip_grad_norm-ClipGradNorm/Square-op12,kernel_graph0,{5967},5967,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5966,6052,0x12c18000ac00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5155,364_Default/clip_grad_norm-ClipGradNorm/SumExt-op12,Default/clip_grad_norm-ClipGradNorm/SumExt-op12,kernel_graph0,{5970-6038},6038,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5967,5970,0x12c18000ae00,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5155,364_Default/clip_grad_norm-ClipGradNorm/SumExt-op12,Default/clip_grad_norm-ClipGradNorm/SumExt-op12,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5972,5977,0x12c224a13800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5158,366_Default/clip_grad_norm-ClipGradNorm/Square-op13,Default/clip_grad_norm-ClipGradNorm/Square-op13,kernel_graph0,{5975},5975,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5974,6053,0x12c18000ae00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5159,367_Default/clip_grad_norm-ClipGradNorm/SumExt-op13,Default/clip_grad_norm-ClipGradNorm/SumExt-op13,kernel_graph0,{5978-6038},6038,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5975,5978,0x12c18000b000,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5159,367_Default/clip_grad_norm-ClipGradNorm/SumExt-op13,Default/clip_grad_norm-ClipGradNorm/SumExt-op13,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5980,5985,0x12c224a13800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5162,369_Default/clip_grad_norm-ClipGradNorm/Square-op14,Default/clip_grad_norm-ClipGradNorm/Square-op14,kernel_graph0,{5983},5983,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5982,6054,0x12c18000b000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5163,370_Default/clip_grad_norm-ClipGradNorm/SumExt-op14,Default/clip_grad_norm-ClipGradNorm/SumExt-op14,kernel_graph0,{5986-6038},6038,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5983,5986,0x12c18000b200,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5163,370_Default/clip_grad_norm-ClipGradNorm/SumExt-op14,Default/clip_grad_norm-ClipGradNorm/SumExt-op14,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5988,5993,0x12c1a6d31c00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5165,372_Default/clip_grad_norm-ClipGradNorm/Square-op15,Default/clip_grad_norm-ClipGradNorm/Square-op15,kernel_graph0,{5991},5991,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5990,6055,0x12c18000b200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5166,373_Default/clip_grad_norm-ClipGradNorm/SumExt-op15,Default/clip_grad_norm-ClipGradNorm/SumExt-op15,kernel_graph0,{5994-6038},6038,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5991,5994,0x12c18000b400,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5166,373_Default/clip_grad_norm-ClipGradNorm/SumExt-op15,Default/clip_grad_norm-ClipGradNorm/SumExt-op15,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5996,6001,0x12c18000b400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5167,375_Default/clip_grad_norm-ClipGradNorm/Square-op16,Default/clip_grad_norm-ClipGradNorm/Square-op16,kernel_graph0,{5999},5999,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5998,6056,0x12c18000f600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5169,376_Default/clip_grad_norm-ClipGradNorm/SumExt-op16,Default/clip_grad_norm-ClipGradNorm/SumExt-op16,kernel_graph0,{6002-6038},6038,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 5999,6002,0x12c18000f800,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5169,376_Default/clip_grad_norm-ClipGradNorm/SumExt-op16,Default/clip_grad_norm-ClipGradNorm/SumExt-op16,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6004,6009,0x12c18000b400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5170,378_Default/clip_grad_norm-ClipGradNorm/Square-op17,Default/clip_grad_norm-ClipGradNorm/Square-op17,kernel_graph0,{6007},6007,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6006,6057,0x12c18000f800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5172,379_Default/clip_grad_norm-ClipGradNorm/SumExt-op17,Default/clip_grad_norm-ClipGradNorm/SumExt-op17,kernel_graph0,{6010-6038},6038,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6007,6010,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5172,379_Default/clip_grad_norm-ClipGradNorm/SumExt-op17,Default/clip_grad_norm-ClipGradNorm/SumExt-op17,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6012,6017,0x12c18000b400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5173,381_Default/clip_grad_norm-ClipGradNorm/Square-op18,Default/clip_grad_norm-ClipGradNorm/Square-op18,kernel_graph0,{6015},6015,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6014,6058,0x12c18000fa00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5176,382_Default/clip_grad_norm-ClipGradNorm/SumExt-op18,Default/clip_grad_norm-ClipGradNorm/SumExt-op18,kernel_graph0,{6018-6038},6038,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6015,6018,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5176,382_Default/clip_grad_norm-ClipGradNorm/SumExt-op18,Default/clip_grad_norm-ClipGradNorm/SumExt-op18,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6020,6025,0x12c18000b400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5177,384_Default/clip_grad_norm-ClipGradNorm/Square-op19,Default/clip_grad_norm-ClipGradNorm/Square-op19,kernel_graph0,{6023},6023,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6022,6059,0x12c18000fc00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5179,385_Default/clip_grad_norm-ClipGradNorm/SumExt-op19,Default/clip_grad_norm-ClipGradNorm/SumExt-op19,kernel_graph0,{6026-6038},6038,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6023,6026,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5179,385_Default/clip_grad_norm-ClipGradNorm/SumExt-op19,Default/clip_grad_norm-ClipGradNorm/SumExt-op19,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6028,9393,0x12c18000b400,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5180,387_Default/AllFinite-op0,Default/AllFinite-op0,kernel_graph0,{6064-6067-6069},6069,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6030,6035,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5182,388_Default/clip_grad_norm-ClipGradNorm/Square-op20,Default/clip_grad_norm-ClipGradNorm/Square-op20,kernel_graph0,{6033},6033,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6032,6060,0x12c18000b600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5183,389_Default/clip_grad_norm-ClipGradNorm/SumExt-op20,Default/clip_grad_norm-ClipGradNorm/SumExt-op20,kernel_graph0,{6036-6038},6038,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6033,6036,0x12c18000b800,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5183,389_Default/clip_grad_norm-ClipGradNorm/SumExt-op20,Default/clip_grad_norm-ClipGradNorm/SumExt-op20,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6038,6063,0x12c18000b800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5184,391_Default/clip_grad_norm-ClipGradNorm/AddN-op0,Default/clip_grad_norm-ClipGradNorm/AddN-op0,kernel_graph0,{6061},6061,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6061,9390,0x12c18000b600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5185,392_Default/clip_grad_norm-ClipGradNorm/Sqrt-op0,Default/clip_grad_norm-ClipGradNorm/Sqrt-op0,kernel_graph0,{6071-6073},6073,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6064,6094,0x12c180008a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5186,393_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Select-op1,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Select-op1,kernel_graph0,{6085-6090},6090,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6067,6211,0x12c180008c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5186,394_Default/LogicalNot-op0,Default/LogicalNot-op0,kernel_graph0,{6087},6087,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6069,6097,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5187,395_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/LogicalOr-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/LogicalOr-op0,kernel_graph0,{6095},6095,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6071,6075,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5188,396_Default/clip_grad_norm-ClipGradNorm/GreaterEqual-op0,Default/clip_grad_norm-ClipGradNorm/GreaterEqual-op0,kernel_graph0,{6073},6073,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6073,6078,0x12c180009600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5188,397_Default/clip_grad_norm-ClipGradNorm/Select-op0,Default/clip_grad_norm-ClipGradNorm/Select-op0,kernel_graph0,{6076},6076,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6076,6081,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5188,398_Default/learning_rate-CosineWithWarmUpLR/Add-op0,Default/learning_rate-CosineWithWarmUpLR/Add-op0,kernel_graph0,{6079},6079,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6079,6084,0x12c180009600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5188,399_Default/clip_grad_norm-ClipGradNorm/Div-op0,Default/clip_grad_norm-ClipGradNorm/Div-op0,kernel_graph0,{6082},6082,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6082,6163,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5189,400_Default/clip_grad_norm-ClipGradNorm/Minimum-op0,Default/clip_grad_norm-ClipGradNorm/Minimum-op0,kernel_graph0,{6100-6103-6106-6109-6112-6115-6118-6121-6124-6127-6130-6133-6136-6139-6142-6145-6148-6151-6154-6157-6160},6160,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6085,6093,0x12c180009600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5189,401_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Mul-op1,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Mul-op1,kernel_graph0,{6090},6090,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6087,6092,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5190,402_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/LogicalAnd-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/LogicalAnd-op0,kernel_graph0,{6090},6090,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6090,6169,0x12c180009200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5190,403_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Select-op2,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Select-op2,kernel_graph0,{6167},6167,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6095,6165,0x12c180008a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5193,404_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Select-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Select-op0,kernel_graph0,{6163},6163,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6098,6167,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5194,405_Default/learning_rate-CosineWithWarmUpLR/Add-op1,Default/learning_rate-CosineWithWarmUpLR/Add-op1,kernel_graph0,{6165},6165,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6100,6370,0x12c1a6d31c00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5195,406_Default/clip_grad_norm-ClipGradNorm/Mul-op0,Default/clip_grad_norm-ClipGradNorm/Mul-op0,kernel_graph0,{6366-6368},6368,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6103,6360,0x12c209e12c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5195,407_Default/clip_grad_norm-ClipGradNorm/Mul-op1,Default/clip_grad_norm-ClipGradNorm/Mul-op1,kernel_graph0,{6356-6358},6358,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6106,6352,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5195,408_Default/clip_grad_norm-ClipGradNorm/Mul-op2,Default/clip_grad_norm-ClipGradNorm/Mul-op2,kernel_graph0,{6348-6350},6350,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6109,6344,0x12c1e9c12000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5195,409_Default/clip_grad_norm-ClipGradNorm/Mul-op3,Default/clip_grad_norm-ClipGradNorm/Mul-op3,kernel_graph0,{6340-6342},6342,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6112,6336,0x12c1f5c12600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5196,410_Default/clip_grad_norm-ClipGradNorm/Mul-op4,Default/clip_grad_norm-ClipGradNorm/Mul-op4,kernel_graph0,{6332-6334},6334,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6115,6328,0x12c1c6131e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5198,411_Default/clip_grad_norm-ClipGradNorm/Mul-op5,Default/clip_grad_norm-ClipGradNorm/Mul-op5,kernel_graph0,{6324-6326},6326,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6118,6320,0x12c1d0d32000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5199,412_Default/clip_grad_norm-ClipGradNorm/Mul-op6,Default/clip_grad_norm-ClipGradNorm/Mul-op6,kernel_graph0,{6316-6318},6318,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6121,6312,0x12c20de12e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5199,413_Default/clip_grad_norm-ClipGradNorm/Mul-op7,Default/clip_grad_norm-ClipGradNorm/Mul-op7,kernel_graph0,{6308-6310},6310,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6124,6304,0x12c23d614400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5201,414_Default/clip_grad_norm-ClipGradNorm/Mul-op8,Default/clip_grad_norm-ClipGradNorm/Mul-op8,kernel_graph0,{6300-6302},6302,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6127,6296,0x12c1db932200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5202,415_Default/clip_grad_norm-ClipGradNorm/Mul-op9,Default/clip_grad_norm-ClipGradNorm/Mul-op9,kernel_graph0,{6292-6294},6294,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6130,6288,0x12c1edc12200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5203,416_Default/clip_grad_norm-ClipGradNorm/Mul-op10,Default/clip_grad_norm-ClipGradNorm/Mul-op10,kernel_graph0,{6284-6286},6286,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6133,6280,0x12c1df932400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5204,417_Default/clip_grad_norm-ClipGradNorm/Mul-op11,Default/clip_grad_norm-ClipGradNorm/Mul-op11,kernel_graph0,{6276-6278},6278,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6136,6272,0x12c224a13800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5204,418_Default/clip_grad_norm-ClipGradNorm/Mul-op12,Default/clip_grad_norm-ClipGradNorm/Mul-op12,kernel_graph0,{6268-6270},6270,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6139,6264,0x12c22f613a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5206,419_Default/clip_grad_norm-ClipGradNorm/Mul-op13,Default/clip_grad_norm-ClipGradNorm/Mul-op13,kernel_graph0,{6260-6262},6262,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6142,6256,0x12c218a13000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5207,420_Default/clip_grad_norm-ClipGradNorm/Mul-op14,Default/clip_grad_norm-ClipGradNorm/Mul-op14,kernel_graph0,{6252-6254},6254,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6145,6248,0x12c18001c800,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5207,421_Default/clip_grad_norm-ClipGradNorm/Mul-op15,Default/clip_grad_norm-ClipGradNorm/Mul-op15,kernel_graph0,{6244-6246},6246,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6148,6240,0x12c18000b800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5208,422_Default/clip_grad_norm-ClipGradNorm/Mul-op16,Default/clip_grad_norm-ClipGradNorm/Mul-op16,kernel_graph0,{6236-6238},6238,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6151,6232,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5209,423_Default/clip_grad_norm-ClipGradNorm/Mul-op17,Default/clip_grad_norm-ClipGradNorm/Mul-op17,kernel_graph0,{6228-6230},6230,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6154,6224,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5210,424_Default/clip_grad_norm-ClipGradNorm/Mul-op18,Default/clip_grad_norm-ClipGradNorm/Mul-op18,kernel_graph0,{6220-6222},6222,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6157,6216,0x12c180018600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5211,425_Default/clip_grad_norm-ClipGradNorm/Mul-op19,Default/clip_grad_norm-ClipGradNorm/Mul-op19,kernel_graph0,{6212-6214},6214,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6160,6378,0x12c180010200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5211,426_Default/clip_grad_norm-ClipGradNorm/Mul-op20,Default/clip_grad_norm-ClipGradNorm/Mul-op20,kernel_graph0,{6374-6376},6376,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6170,6179,0x12c180008a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5217,430_Default/learning_rate-CosineWithWarmUpLR/Cast-op0,Default/learning_rate-CosineWithWarmUpLR/Cast-op0,kernel_graph0,{6172-6175-6177},6177,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6172,6174,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,5218,431_Default/learning_rate-CosineWithWarmUpLR/GreaterEqual-op0,Default/learning_rate-CosineWithWarmUpLR/GreaterEqual-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6175,6180,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6174,435_Default/learning_rate-CosineWithWarmUpLR/Greater-op0,Default/learning_rate-CosineWithWarmUpLR/Greater-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6177,6183,0x12c18000fa00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6175,436_Default/TensorMove-op0,Default/TensorMove-op0,kernel_graph0,{6181},6181,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6181,6186,0x12c180008a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6180,440_Default/learning_rate-CosineWithWarmUpLR/Sub-op1,Default/learning_rate-CosineWithWarmUpLR/Sub-op1,kernel_graph0,{6184},6184,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6184,6189,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6181,441_Default/learning_rate-CosineWithWarmUpLR/Div-op1,Default/learning_rate-CosineWithWarmUpLR/Div-op1,kernel_graph0,{6187},6187,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6187,6192,0x12c180008a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6182,442_Default/learning_rate-CosineWithWarmUpLR/Mul-op1,Default/learning_rate-CosineWithWarmUpLR/Mul-op1,kernel_graph0,{6190},6190,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6190,6195,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6182,443_Default/learning_rate-CosineWithWarmUpLR/Cos-op0,Default/learning_rate-CosineWithWarmUpLR/Cos-op0,kernel_graph0,{6193},6193,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6193,6198,0x12c180008a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6184,444_Default/learning_rate-CosineWithWarmUpLR/Add-op3,Default/learning_rate-CosineWithWarmUpLR/Add-op3,kernel_graph0,{6196},6196,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6196,6201,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6185,445_Default/learning_rate-CosineWithWarmUpLR/Mul-op2,Default/learning_rate-CosineWithWarmUpLR/Mul-op2,kernel_graph0,{6199},6199,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6199,6204,0x12c180008a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6185,446_Default/learning_rate-CosineWithWarmUpLR/Maximum-op0,Default/learning_rate-CosineWithWarmUpLR/Maximum-op0,kernel_graph0,{6202},6202,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6202,6207,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6187,447_Default/learning_rate-CosineWithWarmUpLR/Mul-op3,Default/learning_rate-CosineWithWarmUpLR/Mul-op3,kernel_graph0,{6205},6205,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6205,6210,0x12c180008a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6187,448_Default/learning_rate-CosineWithWarmUpLR/Add-op4,Default/learning_rate-CosineWithWarmUpLR/Add-op4,kernel_graph0,{6208},6208,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6208,9391,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6191,451_Default/Reshape-op0,Default/Reshape-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6212,7041,0x12c180014400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6211,453_Default/optimizer-FP32StateAdamWeightDecay/Mul-op96,Default/optimizer-FP32StateAdamWeightDecay/Mul-op96,kernel_graph0,{7038},7038,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6214,6219,0x12c223613200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6212,454_Default/optimizer-FP32StateAdamWeightDecay/Square-op19,Default/optimizer-FP32StateAdamWeightDecay/Square-op19,kernel_graph0,{6217},6217,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6217,7037,0x12c180018600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6213,455_Default/optimizer-FP32StateAdamWeightDecay/Mul-op98,Default/optimizer-FP32StateAdamWeightDecay/Mul-op98,kernel_graph0,{7034},7034,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6220,7005,0x12c223613200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6213,456_Default/optimizer-FP32StateAdamWeightDecay/Mul-op91,Default/optimizer-FP32StateAdamWeightDecay/Mul-op91,kernel_graph0,{7002},7002,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6222,6227,0x12c223617400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6214,457_Default/optimizer-FP32StateAdamWeightDecay/Square-op18,Default/optimizer-FP32StateAdamWeightDecay/Square-op18,kernel_graph0,{6225},6225,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6225,7009,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6215,458_Default/optimizer-FP32StateAdamWeightDecay/Mul-op93,Default/optimizer-FP32StateAdamWeightDecay/Mul-op93,kernel_graph0,{7006},7006,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6228,6973,0x12c223617400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6215,459_Default/optimizer-FP32StateAdamWeightDecay/Mul-op86,Default/optimizer-FP32StateAdamWeightDecay/Mul-op86,kernel_graph0,{6970},6970,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6230,6235,0x12c22361b600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6217,460_Default/optimizer-FP32StateAdamWeightDecay/Square-op17,Default/optimizer-FP32StateAdamWeightDecay/Square-op17,kernel_graph0,{6233},6233,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6233,6977,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6218,461_Default/optimizer-FP32StateAdamWeightDecay/Mul-op88,Default/optimizer-FP32StateAdamWeightDecay/Mul-op88,kernel_graph0,{6974},6974,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6236,6941,0x12c22361b600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6218,462_Default/optimizer-FP32StateAdamWeightDecay/Mul-op81,Default/optimizer-FP32StateAdamWeightDecay/Mul-op81,kernel_graph0,{6938},6938,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6238,6243,0x12c22361f800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6220,463_Default/optimizer-FP32StateAdamWeightDecay/Square-op16,Default/optimizer-FP32StateAdamWeightDecay/Square-op16,kernel_graph0,{6241},6241,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6241,6945,0x12c18000b800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6221,464_Default/optimizer-FP32StateAdamWeightDecay/Mul-op83,Default/optimizer-FP32StateAdamWeightDecay/Mul-op83,kernel_graph0,{6942},6942,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6244,6909,0x12c241614600,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6221,465_Default/optimizer-FP32StateAdamWeightDecay/Mul-op76,Default/optimizer-FP32StateAdamWeightDecay/Mul-op76,kernel_graph0,{6906},6906,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6246,6251,0x12c260a14800,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6222,466_Default/optimizer-FP32StateAdamWeightDecay/Square-op15,Default/optimizer-FP32StateAdamWeightDecay/Square-op15,kernel_graph0,{6249},6249,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6249,6913,0x12c18001c800,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6223,467_Default/optimizer-FP32StateAdamWeightDecay/Mul-op78,Default/optimizer-FP32StateAdamWeightDecay/Mul-op78,kernel_graph0,{6910},6910,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6252,6877,0x12c1f9c12800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6223,468_Default/optimizer-FP32StateAdamWeightDecay/Mul-op71,Default/optimizer-FP32StateAdamWeightDecay/Mul-op71,kernel_graph0,{6874},6874,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6254,6259,0x12c260a14800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6225,469_Default/optimizer-FP32StateAdamWeightDecay/Square-op14,Default/optimizer-FP32StateAdamWeightDecay/Square-op14,kernel_graph0,{6257},6257,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6257,6881,0x12c218a13000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6226,470_Default/optimizer-FP32StateAdamWeightDecay/Mul-op73,Default/optimizer-FP32StateAdamWeightDecay/Mul-op73,kernel_graph0,{6878},6878,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6260,6845,0x12c260a14800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6226,471_Default/optimizer-FP32StateAdamWeightDecay/Mul-op66,Default/optimizer-FP32StateAdamWeightDecay/Mul-op66,kernel_graph0,{6842},6842,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6262,6267,0x12c26b614a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6228,472_Default/optimizer-FP32StateAdamWeightDecay/Square-op13,Default/optimizer-FP32StateAdamWeightDecay/Square-op13,kernel_graph0,{6265},6265,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6265,6849,0x12c22f613a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6229,473_Default/optimizer-FP32StateAdamWeightDecay/Mul-op68,Default/optimizer-FP32StateAdamWeightDecay/Mul-op68,kernel_graph0,{6846},6846,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6268,6813,0x12c26b614a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6229,474_Default/optimizer-FP32StateAdamWeightDecay/Mul-op61,Default/optimizer-FP32StateAdamWeightDecay/Mul-op61,kernel_graph0,{6810},6810,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6270,6275,0x12c276214c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6230,475_Default/optimizer-FP32StateAdamWeightDecay/Square-op12,Default/optimizer-FP32StateAdamWeightDecay/Square-op12,kernel_graph0,{6273},6273,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6273,6817,0x12c22361f800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6231,476_Default/optimizer-FP32StateAdamWeightDecay/Mul-op63,Default/optimizer-FP32StateAdamWeightDecay/Mul-op63,kernel_graph0,{6814},6814,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6276,6781,0x12c204812a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6231,477_Default/optimizer-FP32StateAdamWeightDecay/Mul-op56,Default/optimizer-FP32StateAdamWeightDecay/Mul-op56,kernel_graph0,{6778},6778,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6278,6283,0x12c1e3932600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6233,478_Default/optimizer-FP32StateAdamWeightDecay/Square-op11,Default/optimizer-FP32StateAdamWeightDecay/Square-op11,kernel_graph0,{6281},6281,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6281,6785,0x12c1df932400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6234,479_Default/optimizer-FP32StateAdamWeightDecay/Mul-op58,Default/optimizer-FP32StateAdamWeightDecay/Mul-op58,kernel_graph0,{6782},6782,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6284,6749,0x12c1e3932600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6234,480_Default/optimizer-FP32StateAdamWeightDecay/Mul-op51,Default/optimizer-FP32StateAdamWeightDecay/Mul-op51,kernel_graph0,{6746},6746,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6286,6291,0x12c19f41ca00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6236,481_Default/optimizer-FP32StateAdamWeightDecay/Square-op10,Default/optimizer-FP32StateAdamWeightDecay/Square-op10,kernel_graph0,{6289},6289,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6289,6753,0x12c1edc12200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6237,482_Default/optimizer-FP32StateAdamWeightDecay/Mul-op53,Default/optimizer-FP32StateAdamWeightDecay/Mul-op53,kernel_graph0,{6750},6750,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6292,6717,0x12c19f41ca00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6237,483_Default/optimizer-FP32StateAdamWeightDecay/Mul-op46,Default/optimizer-FP32StateAdamWeightDecay/Mul-op46,kernel_graph0,{6714},6714,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6294,6299,0x12c276214c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6238,484_Default/optimizer-FP32StateAdamWeightDecay/Square-op9,Default/optimizer-FP32StateAdamWeightDecay/Square-op9,kernel_graph0,{6297},6297,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6297,6721,0x12c1db932200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6239,485_Default/optimizer-FP32StateAdamWeightDecay/Mul-op48,Default/optimizer-FP32StateAdamWeightDecay/Mul-op48,kernel_graph0,{6718},6718,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6300,6685,0x12c276214c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6239,486_Default/optimizer-FP32StateAdamWeightDecay/Mul-op41,Default/optimizer-FP32StateAdamWeightDecay/Mul-op41,kernel_graph0,{6682},6682,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6302,6307,0x12c27a214e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6241,487_Default/optimizer-FP32StateAdamWeightDecay/Square-op8,Default/optimizer-FP32StateAdamWeightDecay/Square-op8,kernel_graph0,{6305},6305,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6305,6689,0x12c23a213c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6242,488_Default/optimizer-FP32StateAdamWeightDecay/Mul-op43,Default/optimizer-FP32StateAdamWeightDecay/Mul-op43,kernel_graph0,{6686},6686,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6308,6653,0x12c27a214e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6242,489_Default/optimizer-FP32StateAdamWeightDecay/Mul-op36,Default/optimizer-FP32StateAdamWeightDecay/Mul-op36,kernel_graph0,{6650},6650,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6310,6315,0x12c284e15000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6244,490_Default/optimizer-FP32StateAdamWeightDecay/Square-op7,Default/optimizer-FP32StateAdamWeightDecay/Square-op7,kernel_graph0,{6313},6313,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6313,6657,0x12c20de12e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6245,491_Default/optimizer-FP32StateAdamWeightDecay/Mul-op38,Default/optimizer-FP32StateAdamWeightDecay/Mul-op38,kernel_graph0,{6654},6654,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6316,6621,0x12c284e15000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6245,492_Default/optimizer-FP32StateAdamWeightDecay/Mul-op31,Default/optimizer-FP32StateAdamWeightDecay/Mul-op31,kernel_graph0,{6618},6618,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6318,6323,0x12c28fa15200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6246,493_Default/optimizer-FP32StateAdamWeightDecay/Square-op6,Default/optimizer-FP32StateAdamWeightDecay/Square-op6,kernel_graph0,{6321},6321,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6321,6625,0x12c1d0d32000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6247,494_Default/optimizer-FP32StateAdamWeightDecay/Mul-op33,Default/optimizer-FP32StateAdamWeightDecay/Mul-op33,kernel_graph0,{6622},6622,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6324,6589,0x12c28fa15200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6247,495_Default/optimizer-FP32StateAdamWeightDecay/Mul-op26,Default/optimizer-FP32StateAdamWeightDecay/Mul-op26,kernel_graph0,{6586},6586,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6326,6331,0x12c29a615400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6249,496_Default/optimizer-FP32StateAdamWeightDecay/Square-op5,Default/optimizer-FP32StateAdamWeightDecay/Square-op5,kernel_graph0,{6329},6329,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6329,6593,0x12c1c6131e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6250,497_Default/optimizer-FP32StateAdamWeightDecay/Mul-op28,Default/optimizer-FP32StateAdamWeightDecay/Mul-op28,kernel_graph0,{6590},6590,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6332,6557,0x12c29a615400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6250,498_Default/optimizer-FP32StateAdamWeightDecay/Mul-op21,Default/optimizer-FP32StateAdamWeightDecay/Mul-op21,kernel_graph0,{6554},6554,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6334,6339,0x12c29e615600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6252,499_Default/optimizer-FP32StateAdamWeightDecay/Square-op4,Default/optimizer-FP32StateAdamWeightDecay/Square-op4,kernel_graph0,{6337},6337,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6337,6561,0x12c1f5c12600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6253,500_Default/optimizer-FP32StateAdamWeightDecay/Mul-op23,Default/optimizer-FP32StateAdamWeightDecay/Mul-op23,kernel_graph0,{6558},6558,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6340,6525,0x12c29e615600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6253,501_Default/optimizer-FP32StateAdamWeightDecay/Mul-op16,Default/optimizer-FP32StateAdamWeightDecay/Mul-op16,kernel_graph0,{6522},6522,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6342,6347,0x12c2a2615800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6254,502_Default/optimizer-FP32StateAdamWeightDecay/Square-op3,Default/optimizer-FP32StateAdamWeightDecay/Square-op3,kernel_graph0,{6345},6345,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6345,6529,0x12c1e7932800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6255,503_Default/optimizer-FP32StateAdamWeightDecay/Mul-op18,Default/optimizer-FP32StateAdamWeightDecay/Mul-op18,kernel_graph0,{6526},6526,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6348,6493,0x12c2a2615800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6257,504_Default/optimizer-FP32StateAdamWeightDecay/Mul-op11,Default/optimizer-FP32StateAdamWeightDecay/Mul-op11,kernel_graph0,{6490},6490,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6350,6355,0x12c2a6615a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6257,505_Default/optimizer-FP32StateAdamWeightDecay/Square-op2,Default/optimizer-FP32StateAdamWeightDecay/Square-op2,kernel_graph0,{6353},6353,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6353,6497,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6258,506_Default/optimizer-FP32StateAdamWeightDecay/Mul-op13,Default/optimizer-FP32StateAdamWeightDecay/Mul-op13,kernel_graph0,{6494},6494,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6356,6461,0x12c2a6615a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6259,507_Default/optimizer-FP32StateAdamWeightDecay/Mul-op6,Default/optimizer-FP32StateAdamWeightDecay/Mul-op6,kernel_graph0,{6458},6458,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6358,6363,0x12c2aa615c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6260,508_Default/optimizer-FP32StateAdamWeightDecay/Square-op1,Default/optimizer-FP32StateAdamWeightDecay/Square-op1,kernel_graph0,{6361},6361,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6361,6465,0x12c208812c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6261,509_Default/optimizer-FP32StateAdamWeightDecay/Mul-op8,Default/optimizer-FP32StateAdamWeightDecay/Mul-op8,kernel_graph0,{6462},6462,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6364,6389,0x12c180008a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6261,510_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Cast-op0,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Cast-op0,kernel_graph0,{6382-6385-6387},6387,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6366,6429,0x12c2aa615c00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6262,511_Default/optimizer-FP32StateAdamWeightDecay/Mul-op1,Default/optimizer-FP32StateAdamWeightDecay/Mul-op1,kernel_graph0,{6426},6426,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6368,6373,0x12c2c9a15e00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6263,512_Default/optimizer-FP32StateAdamWeightDecay/Square-op0,Default/optimizer-FP32StateAdamWeightDecay/Square-op0,kernel_graph0,{6371},6371,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6371,6433,0x12c1a341cc00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6264,513_Default/optimizer-FP32StateAdamWeightDecay/Mul-op3,Default/optimizer-FP32StateAdamWeightDecay/Mul-op3,kernel_graph0,{6430},6430,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6374,7073,0x12c22e21fa00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6265,514_Default/optimizer-FP32StateAdamWeightDecay/Mul-op101,Default/optimizer-FP32StateAdamWeightDecay/Mul-op101,kernel_graph0,{7070},7070,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6376,6381,0x12c22e223c00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6266,515_Default/optimizer-FP32StateAdamWeightDecay/Square-op20,Default/optimizer-FP32StateAdamWeightDecay/Square-op20,kernel_graph0,{6379},6379,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6379,7069,0x12c180010200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6267,516_Default/optimizer-FP32StateAdamWeightDecay/Mul-op103,Default/optimizer-FP32StateAdamWeightDecay/Mul-op103,kernel_graph0,{7066},7066,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6382,6384,0x12c180008c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6268,518_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/GreaterEqual-op0,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/GreaterEqual-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6385,6390,0x12c180008c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6384,522_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Greater-op0,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Greater-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6387,6393,0x12c18000fa00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6385,523_Default/TensorMove-op1,Default/TensorMove-op1,kernel_graph0,{6391},6391,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6391,6396,0x12c18000fc00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6390,527_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Sub-op1,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Sub-op1,kernel_graph0,{6394},6394,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6394,6399,0x12c18000fa00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6391,528_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Div-op1,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Div-op1,kernel_graph0,{6397},6397,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6397,6402,0x12c18000fc00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6392,529_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Mul-op1,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Mul-op1,kernel_graph0,{6400},6400,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6400,6405,0x12c18000fa00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6392,530_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Cos-op0,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Cos-op0,kernel_graph0,{6403},6403,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6403,6408,0x12c18000fc00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6394,531_Default/learning_rate-CosineWithWarmUpLR/Add-op6,Default/learning_rate-CosineWithWarmUpLR/Add-op6,kernel_graph0,{6406},6406,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6406,6411,0x12c18000fa00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6395,532_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Mul-op2,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Mul-op2,kernel_graph0,{6409},6409,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6409,6414,0x12c18000fc00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6395,533_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Maximum-op0,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Maximum-op0,kernel_graph0,{6412},6412,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6412,6417,0x12c18000fa00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6397,534_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Mul-op3,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Mul-op3,kernel_graph0,{6415},6415,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6415,6420,0x12c18000fc00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6398,535_Default/learning_rate-CosineWithWarmUpLR/Add-op7,Default/learning_rate-CosineWithWarmUpLR/Add-op7,kernel_graph0,{6418},6418,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6418,7084,0x12c18000fa00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6401,538_Default/optimizer-FP32StateAdamWeightDecay/Reshape-op0,Default/optimizer-FP32StateAdamWeightDecay/Reshape-op0,kernel_graph0,{6442-6474-6506-6538-6570-6602-6634-6666-6698-6730-6762-6794-6826-6858-6890-6922-6954-6986-7018-7050-7082},7082,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6422,6428,0x12c2c9a15e00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6403,540_Default/optimizer-FP32StateAdamWeightDecay/Mul-op0,Default/optimizer-FP32StateAdamWeightDecay/Mul-op0,kernel_graph0,{6426},6426,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6424,6432,0x12c2e8e16000,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6404,541_Default/optimizer-FP32StateAdamWeightDecay/Mul-op2,Default/optimizer-FP32StateAdamWeightDecay/Mul-op2,kernel_graph0,{6430},6430,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6426,6451,0x12c308216200,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6404,542_Default/learning_rate-CosineWithWarmUpLR/Add-op8,Default/learning_rate-CosineWithWarmUpLR/Add-op8,kernel_graph0,{6439-6449},6449,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6430,6449,0x12c2aa615c00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6406,543_Default/learning_rate-CosineWithWarmUpLR/Add-op9,Default/learning_rate-CosineWithWarmUpLR/Add-op9,kernel_graph0,{6434-6447},6447,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6434,6438,0x12c1a341cc00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6407,544_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op0,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op0,kernel_graph0,{6436},6436,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6436,6441,0x12c2c9a15e00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6407,545_Default/learning_rate-CosineWithWarmUpLR/Add-op10,Default/learning_rate-CosineWithWarmUpLR/Add-op10,kernel_graph0,{6439},6439,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6439,6444,0x12c2e8e16000,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6409,546_Default/optimizer-FP32StateAdamWeightDecay/Div-op0,Default/optimizer-FP32StateAdamWeightDecay/Div-op0,kernel_graph0,{6442},6442,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6442,6447,0x12c2c9a15e00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6410,547_Default/optimizer-FP32StateAdamWeightDecay/Mul-op4,Default/optimizer-FP32StateAdamWeightDecay/Mul-op4,kernel_graph0,{6445},6445,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6445,6453,0x12c2e8e16000,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6410,548_Default/optimizer-FP32StateAdamWeightDecay/Sub-op0,Default/optimizer-FP32StateAdamWeightDecay/Sub-op0,kernel_graph0,{6451},6451,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6454,6460,0x12c1a341cc00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6415,552_Default/optimizer-FP32StateAdamWeightDecay/Mul-op5,Default/optimizer-FP32StateAdamWeightDecay/Mul-op5,kernel_graph0,{6458},6458,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6456,6464,0x12c1a741ce00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6416,553_Default/optimizer-FP32StateAdamWeightDecay/Mul-op7,Default/optimizer-FP32StateAdamWeightDecay/Mul-op7,kernel_graph0,{6462},6462,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6458,6483,0x12c1ab41d000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6416,554_Default/learning_rate-CosineWithWarmUpLR/Add-op11,Default/learning_rate-CosineWithWarmUpLR/Add-op11,kernel_graph0,{6471-6481},6481,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6462,6481,0x12c1a341cc00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6418,555_Default/learning_rate-CosineWithWarmUpLR/Add-op12,Default/learning_rate-CosineWithWarmUpLR/Add-op12,kernel_graph0,{6466-6479},6479,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6466,6470,0x12c1a741ce00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6419,556_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op1,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op1,kernel_graph0,{6468},6468,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6468,6473,0x12c208812c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6420,557_Default/learning_rate-CosineWithWarmUpLR/Add-op13,Default/learning_rate-CosineWithWarmUpLR/Add-op13,kernel_graph0,{6471},6471,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6471,6476,0x12c1a741ce00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6420,558_Default/optimizer-FP32StateAdamWeightDecay/Div-op1,Default/optimizer-FP32StateAdamWeightDecay/Div-op1,kernel_graph0,{6474},6474,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6474,6479,0x12c208812c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6421,559_Default/optimizer-FP32StateAdamWeightDecay/Mul-op9,Default/optimizer-FP32StateAdamWeightDecay/Mul-op9,kernel_graph0,{6477},6477,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6477,6485,0x12c1a741ce00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6421,560_Default/optimizer-FP32StateAdamWeightDecay/Sub-op1,Default/optimizer-FP32StateAdamWeightDecay/Sub-op1,kernel_graph0,{6483},6483,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6486,6492,0x12c208812c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6423,564_Default/optimizer-FP32StateAdamWeightDecay/Mul-op10,Default/optimizer-FP32StateAdamWeightDecay/Mul-op10,kernel_graph0,{6490},6490,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6488,6496,0x12c1a341cc00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6423,565_Default/optimizer-FP32StateAdamWeightDecay/Mul-op12,Default/optimizer-FP32StateAdamWeightDecay/Mul-op12,kernel_graph0,{6494},6494,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6490,6515,0x12c1a741ce00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6424,566_Default/learning_rate-CosineWithWarmUpLR/Add-op14,Default/learning_rate-CosineWithWarmUpLR/Add-op14,kernel_graph0,{6503-6513},6513,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6494,6513,0x12c208812c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6425,567_Default/learning_rate-CosineWithWarmUpLR/Add-op15,Default/learning_rate-CosineWithWarmUpLR/Add-op15,kernel_graph0,{6498-6511},6511,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6498,6502,0x12c1a341cc00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6425,568_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op2,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op2,kernel_graph0,{6500},6500,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6500,6505,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6426,569_Default/learning_rate-CosineWithWarmUpLR/Add-op16,Default/learning_rate-CosineWithWarmUpLR/Add-op16,kernel_graph0,{6503},6503,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6503,6508,0x12c1a341cc00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6427,570_Default/optimizer-FP32StateAdamWeightDecay/Div-op2,Default/optimizer-FP32StateAdamWeightDecay/Div-op2,kernel_graph0,{6506},6506,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6506,6511,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6427,571_Default/optimizer-FP32StateAdamWeightDecay/Mul-op14,Default/optimizer-FP32StateAdamWeightDecay/Mul-op14,kernel_graph0,{6509},6509,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6509,6517,0x12c1a341cc00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6430,572_Default/optimizer-FP32StateAdamWeightDecay/Sub-op2,Default/optimizer-FP32StateAdamWeightDecay/Sub-op2,kernel_graph0,{6515},6515,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6518,6524,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6435,576_Default/optimizer-FP32StateAdamWeightDecay/Mul-op15,Default/optimizer-FP32StateAdamWeightDecay/Mul-op15,kernel_graph0,{6522},6522,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6520,6528,0x12c208812c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6435,577_Default/optimizer-FP32StateAdamWeightDecay/Mul-op17,Default/optimizer-FP32StateAdamWeightDecay/Mul-op17,kernel_graph0,{6526},6526,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6522,6547,0x12c1a341cc00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6436,578_Default/learning_rate-CosineWithWarmUpLR/Add-op17,Default/learning_rate-CosineWithWarmUpLR/Add-op17,kernel_graph0,{6535-6545},6545,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6526,6545,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6437,579_Default/learning_rate-CosineWithWarmUpLR/Add-op18,Default/learning_rate-CosineWithWarmUpLR/Add-op18,kernel_graph0,{6530-6543},6543,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6530,6534,0x12c208812c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6437,580_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op3,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op3,kernel_graph0,{6532},6532,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6532,6537,0x12c1e7932800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6439,581_Default/learning_rate-CosineWithWarmUpLR/Add-op19,Default/learning_rate-CosineWithWarmUpLR/Add-op19,kernel_graph0,{6535},6535,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6535,6540,0x12c208812c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6440,582_Default/optimizer-FP32StateAdamWeightDecay/Div-op3,Default/optimizer-FP32StateAdamWeightDecay/Div-op3,kernel_graph0,{6538},6538,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6538,6543,0x12c1e7932800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6440,583_Default/optimizer-FP32StateAdamWeightDecay/Mul-op19,Default/optimizer-FP32StateAdamWeightDecay/Mul-op19,kernel_graph0,{6541},6541,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6541,6549,0x12c208812c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6442,584_Default/optimizer-FP32StateAdamWeightDecay/Sub-op3,Default/optimizer-FP32StateAdamWeightDecay/Sub-op3,kernel_graph0,{6547},6547,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6550,6556,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6446,588_Default/optimizer-FP32StateAdamWeightDecay/Mul-op20,Default/optimizer-FP32StateAdamWeightDecay/Mul-op20,kernel_graph0,{6554},6554,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6552,6560,0x12c208812c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6446,589_Default/optimizer-FP32StateAdamWeightDecay/Mul-op22,Default/optimizer-FP32StateAdamWeightDecay/Mul-op22,kernel_graph0,{6558},6558,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6554,6579,0x12c1e7932800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6447,590_Default/learning_rate-CosineWithWarmUpLR/Add-op20,Default/learning_rate-CosineWithWarmUpLR/Add-op20,kernel_graph0,{6567-6577},6577,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6558,6577,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6448,591_Default/learning_rate-CosineWithWarmUpLR/Add-op21,Default/learning_rate-CosineWithWarmUpLR/Add-op21,kernel_graph0,{6562-6575},6575,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6562,6566,0x12c1f5c12600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6448,592_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op4,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op4,kernel_graph0,{6564},6564,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6564,6569,0x12c208812c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6448,593_Default/learning_rate-CosineWithWarmUpLR/Add-op22,Default/learning_rate-CosineWithWarmUpLR/Add-op22,kernel_graph0,{6567},6567,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6567,6572,0x12c1f5c12600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6448,594_Default/optimizer-FP32StateAdamWeightDecay/Div-op4,Default/optimizer-FP32StateAdamWeightDecay/Div-op4,kernel_graph0,{6570},6570,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6570,6575,0x12c208812c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6449,595_Default/optimizer-FP32StateAdamWeightDecay/Mul-op24,Default/optimizer-FP32StateAdamWeightDecay/Mul-op24,kernel_graph0,{6573},6573,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6573,6581,0x12c1f5c12600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6449,596_Default/optimizer-FP32StateAdamWeightDecay/Sub-op4,Default/optimizer-FP32StateAdamWeightDecay/Sub-op4,kernel_graph0,{6579},6579,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6582,6588,0x12c1a341cc00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6451,600_Default/optimizer-FP32StateAdamWeightDecay/Mul-op25,Default/optimizer-FP32StateAdamWeightDecay/Mul-op25,kernel_graph0,{6586},6586,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6584,6592,0x12c1ae01ce00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6452,601_Default/optimizer-FP32StateAdamWeightDecay/Mul-op27,Default/optimizer-FP32StateAdamWeightDecay/Mul-op27,kernel_graph0,{6590},6590,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6586,6611,0x12c1b8c1d000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6452,602_Default/learning_rate-CosineWithWarmUpLR/Add-op23,Default/learning_rate-CosineWithWarmUpLR/Add-op23,kernel_graph0,{6599-6609},6609,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6590,6609,0x12c1a341cc00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6452,603_Default/learning_rate-CosineWithWarmUpLR/Add-op24,Default/learning_rate-CosineWithWarmUpLR/Add-op24,kernel_graph0,{6594-6607},6607,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6594,6598,0x12c1ae01ce00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6453,604_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op5,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op5,kernel_graph0,{6596},6596,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6596,6601,0x12c1c381d200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6454,605_Default/learning_rate-CosineWithWarmUpLR/Add-op25,Default/learning_rate-CosineWithWarmUpLR/Add-op25,kernel_graph0,{6599},6599,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6599,6604,0x12c1ae01ce00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6455,606_Default/optimizer-FP32StateAdamWeightDecay/Div-op5,Default/optimizer-FP32StateAdamWeightDecay/Div-op5,kernel_graph0,{6602},6602,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6602,6607,0x12c1c381d200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6455,607_Default/optimizer-FP32StateAdamWeightDecay/Mul-op29,Default/optimizer-FP32StateAdamWeightDecay/Mul-op29,kernel_graph0,{6605},6605,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6605,6613,0x12c1ae01ce00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6456,608_Default/optimizer-FP32StateAdamWeightDecay/Sub-op5,Default/optimizer-FP32StateAdamWeightDecay/Sub-op5,kernel_graph0,{6611},6611,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6614,6620,0x12c1a341cc00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6459,612_Default/optimizer-FP32StateAdamWeightDecay/Mul-op30,Default/optimizer-FP32StateAdamWeightDecay/Mul-op30,kernel_graph0,{6618},6618,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6616,6624,0x12c1ae01ce00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6461,613_Default/optimizer-FP32StateAdamWeightDecay/Mul-op32,Default/optimizer-FP32StateAdamWeightDecay/Mul-op32,kernel_graph0,{6622},6622,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6618,6643,0x12c1b8c1d000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6462,614_Default/learning_rate-CosineWithWarmUpLR/Add-op26,Default/learning_rate-CosineWithWarmUpLR/Add-op26,kernel_graph0,{6631-6641},6641,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6622,6641,0x12c1a341cc00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6463,615_Default/learning_rate-CosineWithWarmUpLR/Add-op27,Default/learning_rate-CosineWithWarmUpLR/Add-op27,kernel_graph0,{6626-6639},6639,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6626,6630,0x12c1ae01ce00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6465,616_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op6,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op6,kernel_graph0,{6628},6628,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6628,6633,0x12c1c381d200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6466,617_Default/learning_rate-CosineWithWarmUpLR/Add-op28,Default/learning_rate-CosineWithWarmUpLR/Add-op28,kernel_graph0,{6631},6631,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6631,6636,0x12c1ae01ce00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6467,618_Default/optimizer-FP32StateAdamWeightDecay/Div-op6,Default/optimizer-FP32StateAdamWeightDecay/Div-op6,kernel_graph0,{6634},6634,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6634,6639,0x12c1c381d200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6467,619_Default/optimizer-FP32StateAdamWeightDecay/Mul-op34,Default/optimizer-FP32StateAdamWeightDecay/Mul-op34,kernel_graph0,{6637},6637,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6637,6645,0x12c1ae01ce00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6468,620_Default/optimizer-FP32StateAdamWeightDecay/Sub-op6,Default/optimizer-FP32StateAdamWeightDecay/Sub-op6,kernel_graph0,{6643},6643,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6646,6652,0x12c1a341cc00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6472,624_Default/optimizer-FP32StateAdamWeightDecay/Mul-op35,Default/optimizer-FP32StateAdamWeightDecay/Mul-op35,kernel_graph0,{6650},6650,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6648,6656,0x12c1ae01ce00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6473,625_Default/optimizer-FP32StateAdamWeightDecay/Mul-op37,Default/optimizer-FP32StateAdamWeightDecay/Mul-op37,kernel_graph0,{6654},6654,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6650,6675,0x12c1b8c1d000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6474,626_Default/learning_rate-CosineWithWarmUpLR/Add-op29,Default/learning_rate-CosineWithWarmUpLR/Add-op29,kernel_graph0,{6663-6673},6673,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6654,6673,0x12c1a341cc00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6475,627_Default/learning_rate-CosineWithWarmUpLR/Add-op30,Default/learning_rate-CosineWithWarmUpLR/Add-op30,kernel_graph0,{6658-6671},6671,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6658,6662,0x12c1ae01ce00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6476,628_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op7,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op7,kernel_graph0,{6660},6660,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6660,6665,0x12c208812c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6477,629_Default/learning_rate-CosineWithWarmUpLR/Add-op31,Default/learning_rate-CosineWithWarmUpLR/Add-op31,kernel_graph0,{6663},6663,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6663,6668,0x12c1ae01ce00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6478,630_Default/optimizer-FP32StateAdamWeightDecay/Div-op7,Default/optimizer-FP32StateAdamWeightDecay/Div-op7,kernel_graph0,{6666},6666,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6666,6671,0x12c208812c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6479,631_Default/optimizer-FP32StateAdamWeightDecay/Mul-op39,Default/optimizer-FP32StateAdamWeightDecay/Mul-op39,kernel_graph0,{6669},6669,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6669,6677,0x12c1ae01ce00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6480,632_Default/optimizer-FP32StateAdamWeightDecay/Sub-op7,Default/optimizer-FP32StateAdamWeightDecay/Sub-op7,kernel_graph0,{6675},6675,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6678,6684,0x12c1e7932800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6482,636_Default/optimizer-FP32StateAdamWeightDecay/Mul-op40,Default/optimizer-FP32StateAdamWeightDecay/Mul-op40,kernel_graph0,{6682},6682,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6680,6688,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6482,637_Default/optimizer-FP32StateAdamWeightDecay/Mul-op42,Default/optimizer-FP32StateAdamWeightDecay/Mul-op42,kernel_graph0,{6686},6686,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6682,6707,0x12c1f5c12600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6483,638_Default/learning_rate-CosineWithWarmUpLR/Add-op32,Default/learning_rate-CosineWithWarmUpLR/Add-op32,kernel_graph0,{6695-6705},6705,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6686,6705,0x12c1e7932800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6484,639_Default/learning_rate-CosineWithWarmUpLR/Add-op33,Default/learning_rate-CosineWithWarmUpLR/Add-op33,kernel_graph0,{6690-6703},6703,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6690,6694,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6484,640_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op8,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op8,kernel_graph0,{6692},6692,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6692,6697,0x12c23a213c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6484,641_Default/learning_rate-CosineWithWarmUpLR/Add-op34,Default/learning_rate-CosineWithWarmUpLR/Add-op34,kernel_graph0,{6695},6695,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6695,6700,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6486,642_Default/optimizer-FP32StateAdamWeightDecay/Div-op8,Default/optimizer-FP32StateAdamWeightDecay/Div-op8,kernel_graph0,{6698},6698,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6698,6703,0x12c23a213c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6486,643_Default/optimizer-FP32StateAdamWeightDecay/Mul-op44,Default/optimizer-FP32StateAdamWeightDecay/Mul-op44,kernel_graph0,{6701},6701,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6701,6709,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6487,644_Default/optimizer-FP32StateAdamWeightDecay/Sub-op8,Default/optimizer-FP32StateAdamWeightDecay/Sub-op8,kernel_graph0,{6707},6707,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6710,6716,0x12c1e7932800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6489,648_Default/optimizer-FP32StateAdamWeightDecay/Mul-op45,Default/optimizer-FP32StateAdamWeightDecay/Mul-op45,kernel_graph0,{6714},6714,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6712,6720,0x12c23a213c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6490,649_Default/optimizer-FP32StateAdamWeightDecay/Mul-op47,Default/optimizer-FP32StateAdamWeightDecay/Mul-op47,kernel_graph0,{6718},6718,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6714,6739,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6491,650_Default/learning_rate-CosineWithWarmUpLR/Add-op35,Default/learning_rate-CosineWithWarmUpLR/Add-op35,kernel_graph0,{6727-6737},6737,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6718,6737,0x12c1f5c12600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6491,651_Default/learning_rate-CosineWithWarmUpLR/Add-op36,Default/learning_rate-CosineWithWarmUpLR/Add-op36,kernel_graph0,{6722-6735},6735,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6722,6726,0x12c1e7932800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6494,652_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op9,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op9,kernel_graph0,{6724},6724,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6724,6729,0x12c23a213c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6495,653_Default/learning_rate-CosineWithWarmUpLR/Add-op37,Default/learning_rate-CosineWithWarmUpLR/Add-op37,kernel_graph0,{6727},6727,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6727,6732,0x12c1e7932800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6495,654_Default/optimizer-FP32StateAdamWeightDecay/Div-op9,Default/optimizer-FP32StateAdamWeightDecay/Div-op9,kernel_graph0,{6730},6730,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6730,6735,0x12c23a213c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6498,655_Default/optimizer-FP32StateAdamWeightDecay/Mul-op49,Default/optimizer-FP32StateAdamWeightDecay/Mul-op49,kernel_graph0,{6733},6733,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6733,6741,0x12c1e7932800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6499,656_Default/optimizer-FP32StateAdamWeightDecay/Sub-op9,Default/optimizer-FP32StateAdamWeightDecay/Sub-op9,kernel_graph0,{6739},6739,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6742,6748,0x12c1e7932800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6501,660_Default/optimizer-FP32StateAdamWeightDecay/Mul-op50,Default/optimizer-FP32StateAdamWeightDecay/Mul-op50,kernel_graph0,{6746},6746,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6744,6752,0x12c23a213c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6503,661_Default/optimizer-FP32StateAdamWeightDecay/Mul-op52,Default/optimizer-FP32StateAdamWeightDecay/Mul-op52,kernel_graph0,{6750},6750,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6746,6771,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6504,662_Default/learning_rate-CosineWithWarmUpLR/Add-op38,Default/learning_rate-CosineWithWarmUpLR/Add-op38,kernel_graph0,{6759-6769},6769,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6750,6769,0x12c1f5c12600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6504,663_Default/learning_rate-CosineWithWarmUpLR/Add-op39,Default/learning_rate-CosineWithWarmUpLR/Add-op39,kernel_graph0,{6754-6767},6767,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6754,6758,0x12c23a213c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6506,664_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op10,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op10,kernel_graph0,{6756},6756,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6756,6761,0x12c1e3932600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6507,665_Default/learning_rate-CosineWithWarmUpLR/Add-op40,Default/learning_rate-CosineWithWarmUpLR/Add-op40,kernel_graph0,{6759},6759,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6759,6764,0x12c23a213c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6507,666_Default/optimizer-FP32StateAdamWeightDecay/Div-op10,Default/optimizer-FP32StateAdamWeightDecay/Div-op10,kernel_graph0,{6762},6762,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6762,6767,0x12c1e3932600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6509,667_Default/optimizer-FP32StateAdamWeightDecay/Mul-op54,Default/optimizer-FP32StateAdamWeightDecay/Mul-op54,kernel_graph0,{6765},6765,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6765,6773,0x12c23a213c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6510,668_Default/optimizer-FP32StateAdamWeightDecay/Sub-op10,Default/optimizer-FP32StateAdamWeightDecay/Sub-op10,kernel_graph0,{6771},6771,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6774,6780,0x12c23a213c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6512,672_Default/optimizer-FP32StateAdamWeightDecay/Mul-op55,Default/optimizer-FP32StateAdamWeightDecay/Mul-op55,kernel_graph0,{6778},6778,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6776,6784,0x12c208812c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6512,673_Default/optimizer-FP32StateAdamWeightDecay/Mul-op57,Default/optimizer-FP32StateAdamWeightDecay/Mul-op57,kernel_graph0,{6782},6782,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6778,6803,0x12c20c812e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6513,674_Default/learning_rate-CosineWithWarmUpLR/Add-op41,Default/learning_rate-CosineWithWarmUpLR/Add-op41,kernel_graph0,{6791-6801},6801,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6782,6801,0x12c204812a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6514,675_Default/learning_rate-CosineWithWarmUpLR/Add-op42,Default/learning_rate-CosineWithWarmUpLR/Add-op42,kernel_graph0,{6786-6799},6799,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6786,6790,0x12c208812c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6514,676_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op11,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op11,kernel_graph0,{6788},6788,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6788,6793,0x12c23a213c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6514,677_Default/learning_rate-CosineWithWarmUpLR/Add-op43,Default/learning_rate-CosineWithWarmUpLR/Add-op43,kernel_graph0,{6791},6791,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6791,6796,0x12c208812c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6515,678_Default/optimizer-FP32StateAdamWeightDecay/Div-op11,Default/optimizer-FP32StateAdamWeightDecay/Div-op11,kernel_graph0,{6794},6794,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6794,6799,0x12c23a213c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6516,679_Default/optimizer-FP32StateAdamWeightDecay/Mul-op59,Default/optimizer-FP32StateAdamWeightDecay/Mul-op59,kernel_graph0,{6797},6797,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6797,6805,0x12c208812c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6516,680_Default/optimizer-FP32StateAdamWeightDecay/Sub-op11,Default/optimizer-FP32StateAdamWeightDecay/Sub-op11,kernel_graph0,{6803},6803,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6806,6812,0x12c204812a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6519,684_Default/optimizer-FP32StateAdamWeightDecay/Mul-op60,Default/optimizer-FP32StateAdamWeightDecay/Mul-op60,kernel_graph0,{6810},6810,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6808,6816,0x12c19f41ca00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6520,685_Default/optimizer-FP32StateAdamWeightDecay/Mul-op62,Default/optimizer-FP32StateAdamWeightDecay/Mul-op62,kernel_graph0,{6814},6814,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6810,6835,0x12c1aa01cc00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6521,686_Default/learning_rate-CosineWithWarmUpLR/Add-op44,Default/learning_rate-CosineWithWarmUpLR/Add-op44,kernel_graph0,{6823-6833},6833,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6814,6833,0x12c204812a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6521,687_Default/learning_rate-CosineWithWarmUpLR/Add-op45,Default/learning_rate-CosineWithWarmUpLR/Add-op45,kernel_graph0,{6818-6831},6831,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6818,6822,0x12c19f41ca00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6522,688_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op12,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op12,kernel_graph0,{6820},6820,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6820,6825,0x12c22361f800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6523,689_Default/learning_rate-CosineWithWarmUpLR/Add-op46,Default/learning_rate-CosineWithWarmUpLR/Add-op46,kernel_graph0,{6823},6823,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6823,6828,0x12c19f41ca00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6523,690_Default/optimizer-FP32StateAdamWeightDecay/Div-op12,Default/optimizer-FP32StateAdamWeightDecay/Div-op12,kernel_graph0,{6826},6826,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6826,6831,0x12c22361f800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6526,691_Default/optimizer-FP32StateAdamWeightDecay/Mul-op64,Default/optimizer-FP32StateAdamWeightDecay/Mul-op64,kernel_graph0,{6829},6829,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6829,6837,0x12c19f41ca00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6527,692_Default/optimizer-FP32StateAdamWeightDecay/Sub-op12,Default/optimizer-FP32StateAdamWeightDecay/Sub-op12,kernel_graph0,{6835},6835,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6838,6844,0x12c22361f800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6531,696_Default/optimizer-FP32StateAdamWeightDecay/Mul-op65,Default/optimizer-FP32StateAdamWeightDecay/Mul-op65,kernel_graph0,{6842},6842,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6840,6848,0x12c204812a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6532,697_Default/optimizer-FP32StateAdamWeightDecay/Mul-op67,Default/optimizer-FP32StateAdamWeightDecay/Mul-op67,kernel_graph0,{6846},6846,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6842,6867,0x12c19f41ca00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6533,698_Default/learning_rate-CosineWithWarmUpLR/Add-op47,Default/learning_rate-CosineWithWarmUpLR/Add-op47,kernel_graph0,{6855-6865},6865,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6846,6865,0x12c22361f800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6533,699_Default/learning_rate-CosineWithWarmUpLR/Add-op48,Default/learning_rate-CosineWithWarmUpLR/Add-op48,kernel_graph0,{6850-6863},6863,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6850,6854,0x12c22e223c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6535,700_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op13,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op13,kernel_graph0,{6852},6852,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6852,6857,0x12c204812a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6536,701_Default/learning_rate-CosineWithWarmUpLR/Add-op49,Default/learning_rate-CosineWithWarmUpLR/Add-op49,kernel_graph0,{6855},6855,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6855,6860,0x12c22e223c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6536,702_Default/optimizer-FP32StateAdamWeightDecay/Div-op13,Default/optimizer-FP32StateAdamWeightDecay/Div-op13,kernel_graph0,{6858},6858,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6858,6863,0x12c204812a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6538,703_Default/optimizer-FP32StateAdamWeightDecay/Mul-op69,Default/optimizer-FP32StateAdamWeightDecay/Mul-op69,kernel_graph0,{6861},6861,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6861,6869,0x12c22e223c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6539,704_Default/optimizer-FP32StateAdamWeightDecay/Sub-op13,Default/optimizer-FP32StateAdamWeightDecay/Sub-op13,kernel_graph0,{6867},6867,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6870,6876,0x12c22361f800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6542,708_Default/optimizer-FP32StateAdamWeightDecay/Mul-op70,Default/optimizer-FP32StateAdamWeightDecay/Mul-op70,kernel_graph0,{6874},6874,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6872,6880,0x12c22e223c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6543,709_Default/optimizer-FP32StateAdamWeightDecay/Mul-op72,Default/optimizer-FP32StateAdamWeightDecay/Mul-op72,kernel_graph0,{6878},6878,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6874,6899,0x12c204812a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6544,710_Default/learning_rate-CosineWithWarmUpLR/Add-op50,Default/learning_rate-CosineWithWarmUpLR/Add-op50,kernel_graph0,{6887-6897},6897,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6878,6897,0x12c22361f800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6544,711_Default/learning_rate-CosineWithWarmUpLR/Add-op51,Default/learning_rate-CosineWithWarmUpLR/Add-op51,kernel_graph0,{6882-6895},6895,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6882,6886,0x12c22e223c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6544,712_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op14,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op14,kernel_graph0,{6884},6884,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6884,6889,0x12c20f412c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6545,713_Default/learning_rate-CosineWithWarmUpLR/Add-op52,Default/learning_rate-CosineWithWarmUpLR/Add-op52,kernel_graph0,{6887},6887,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6887,6892,0x12c22e223c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6546,714_Default/optimizer-FP32StateAdamWeightDecay/Div-op14,Default/optimizer-FP32StateAdamWeightDecay/Div-op14,kernel_graph0,{6890},6890,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6890,6895,0x12c20f412c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6546,715_Default/optimizer-FP32StateAdamWeightDecay/Mul-op74,Default/optimizer-FP32StateAdamWeightDecay/Mul-op74,kernel_graph0,{6893},6893,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6893,6901,0x12c22e223c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6546,716_Default/optimizer-FP32StateAdamWeightDecay/Sub-op14,Default/optimizer-FP32StateAdamWeightDecay/Sub-op14,kernel_graph0,{6899},6899,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6902,6908,0x12c19f41ca00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6548,720_Default/optimizer-FP32StateAdamWeightDecay/Mul-op75,Default/optimizer-FP32StateAdamWeightDecay/Mul-op75,kernel_graph0,{6906},6906,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6904,6912,0x12c1be81cc00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6549,721_Default/optimizer-FP32StateAdamWeightDecay/Mul-op77,Default/optimizer-FP32StateAdamWeightDecay/Mul-op77,kernel_graph0,{6910},6910,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6906,6931,0x12c1ddc1ce00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6550,722_Default/learning_rate-CosineWithWarmUpLR/Add-op53,Default/learning_rate-CosineWithWarmUpLR/Add-op53,kernel_graph0,{6919-6929},6929,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6910,6929,0x12c19f41ca00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6551,723_Default/learning_rate-CosineWithWarmUpLR/Add-op54,Default/learning_rate-CosineWithWarmUpLR/Add-op54,kernel_graph0,{6914-6927},6927,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6914,6918,0x12c18001c800,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6551,724_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op15,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op15,kernel_graph0,{6916},6916,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6916,6921,0x12c1be81cc00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6552,725_Default/learning_rate-CosineWithWarmUpLR/Add-op55,Default/learning_rate-CosineWithWarmUpLR/Add-op55,kernel_graph0,{6919},6919,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6919,6924,0x12c18001c800,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6553,726_Default/optimizer-FP32StateAdamWeightDecay/Div-op15,Default/optimizer-FP32StateAdamWeightDecay/Div-op15,kernel_graph0,{6922},6922,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6922,6927,0x12c1be81cc00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6553,727_Default/optimizer-FP32StateAdamWeightDecay/Mul-op79,Default/optimizer-FP32StateAdamWeightDecay/Mul-op79,kernel_graph0,{6925},6925,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6925,6933,0x12c18001c800,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6554,728_Default/optimizer-FP32StateAdamWeightDecay/Sub-op15,Default/optimizer-FP32StateAdamWeightDecay/Sub-op15,kernel_graph0,{6931},6931,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6934,6940,0x12c22361f800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6559,732_Default/optimizer-FP32StateAdamWeightDecay/Mul-op80,Default/optimizer-FP32StateAdamWeightDecay/Mul-op80,kernel_graph0,{6938},6938,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6936,6944,0x12c223623a00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6561,733_Default/optimizer-FP32StateAdamWeightDecay/Mul-op82,Default/optimizer-FP32StateAdamWeightDecay/Mul-op82,kernel_graph0,{6942},6942,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6938,6963,0x12c223627c00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6562,734_Default/learning_rate-CosineWithWarmUpLR/Add-op56,Default/learning_rate-CosineWithWarmUpLR/Add-op56,kernel_graph0,{6951-6961},6961,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6942,6961,0x12c22361b600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6563,735_Default/learning_rate-CosineWithWarmUpLR/Add-op57,Default/learning_rate-CosineWithWarmUpLR/Add-op57,kernel_graph0,{6946-6959},6959,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6946,6950,0x12c18000b800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6564,736_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op16,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op16,kernel_graph0,{6948},6948,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6948,6953,0x12c22361f800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6564,737_Default/learning_rate-CosineWithWarmUpLR/Add-op58,Default/learning_rate-CosineWithWarmUpLR/Add-op58,kernel_graph0,{6951},6951,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6951,6956,0x12c18000b800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6565,738_Default/optimizer-FP32StateAdamWeightDecay/Div-op16,Default/optimizer-FP32StateAdamWeightDecay/Div-op16,kernel_graph0,{6954},6954,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6954,6959,0x12c22361f800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6565,739_Default/optimizer-FP32StateAdamWeightDecay/Mul-op84,Default/optimizer-FP32StateAdamWeightDecay/Mul-op84,kernel_graph0,{6957},6957,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6957,6965,0x12c18000b800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6567,740_Default/optimizer-FP32StateAdamWeightDecay/Sub-op16,Default/optimizer-FP32StateAdamWeightDecay/Sub-op16,kernel_graph0,{6963},6963,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6966,6972,0x12c18000b800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6571,744_Default/optimizer-FP32StateAdamWeightDecay/Mul-op85,Default/optimizer-FP32StateAdamWeightDecay/Mul-op85,kernel_graph0,{6970},6970,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6968,6976,0x12c22361b600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6571,745_Default/optimizer-FP32StateAdamWeightDecay/Mul-op87,Default/optimizer-FP32StateAdamWeightDecay/Mul-op87,kernel_graph0,{6974},6974,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6970,6995,0x12c22361f800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6573,746_Default/learning_rate-CosineWithWarmUpLR/Add-op59,Default/learning_rate-CosineWithWarmUpLR/Add-op59,kernel_graph0,{6983-6993},6993,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6974,6993,0x12c18000b800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6574,747_Default/learning_rate-CosineWithWarmUpLR/Add-op60,Default/learning_rate-CosineWithWarmUpLR/Add-op60,kernel_graph0,{6978-6991},6991,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6978,6982,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6574,748_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op17,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op17,kernel_graph0,{6980},6980,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6980,6985,0x12c223617400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6575,749_Default/learning_rate-CosineWithWarmUpLR/Add-op61,Default/learning_rate-CosineWithWarmUpLR/Add-op61,kernel_graph0,{6983},6983,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6983,6988,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6576,750_Default/optimizer-FP32StateAdamWeightDecay/Div-op17,Default/optimizer-FP32StateAdamWeightDecay/Div-op17,kernel_graph0,{6986},6986,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6986,6991,0x12c223617400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6576,751_Default/optimizer-FP32StateAdamWeightDecay/Mul-op89,Default/optimizer-FP32StateAdamWeightDecay/Mul-op89,kernel_graph0,{6989},6989,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6989,6997,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6576,752_Default/optimizer-FP32StateAdamWeightDecay/Sub-op17,Default/optimizer-FP32StateAdamWeightDecay/Sub-op17,kernel_graph0,{6995},6995,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 6998,7004,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6578,756_Default/optimizer-FP32StateAdamWeightDecay/Mul-op90,Default/optimizer-FP32StateAdamWeightDecay/Mul-op90,kernel_graph0,{7002},7002,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7000,7008,0x12c18000b800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6578,757_Default/optimizer-FP32StateAdamWeightDecay/Mul-op92,Default/optimizer-FP32StateAdamWeightDecay/Mul-op92,kernel_graph0,{7006},7006,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7002,7027,0x12c223617400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6579,758_Default/learning_rate-CosineWithWarmUpLR/Add-op62,Default/learning_rate-CosineWithWarmUpLR/Add-op62,kernel_graph0,{7015-7025},7025,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7006,7025,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6580,759_Default/learning_rate-CosineWithWarmUpLR/Add-op63,Default/learning_rate-CosineWithWarmUpLR/Add-op63,kernel_graph0,{7010-7023},7023,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7010,7014,0x12c18000b800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6580,760_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op18,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op18,kernel_graph0,{7012},7012,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7012,7017,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6580,761_Default/learning_rate-CosineWithWarmUpLR/Add-op64,Default/learning_rate-CosineWithWarmUpLR/Add-op64,kernel_graph0,{7015},7015,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7015,7020,0x12c18000b800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6581,762_Default/optimizer-FP32StateAdamWeightDecay/Div-op18,Default/optimizer-FP32StateAdamWeightDecay/Div-op18,kernel_graph0,{7018},7018,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7018,7023,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6582,763_Default/optimizer-FP32StateAdamWeightDecay/Mul-op94,Default/optimizer-FP32StateAdamWeightDecay/Mul-op94,kernel_graph0,{7021},7021,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7021,7029,0x12c18000b800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6583,764_Default/optimizer-FP32StateAdamWeightDecay/Sub-op18,Default/optimizer-FP32StateAdamWeightDecay/Sub-op18,kernel_graph0,{7027},7027,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7030,7036,0x12c18000b800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6585,768_Default/optimizer-FP32StateAdamWeightDecay/Mul-op97,Default/optimizer-FP32StateAdamWeightDecay/Mul-op97,kernel_graph0,{7034},7034,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7032,7040,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6586,769_Default/optimizer-FP32StateAdamWeightDecay/Mul-op95,Default/optimizer-FP32StateAdamWeightDecay/Mul-op95,kernel_graph0,{7038},7038,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7034,7057,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6587,770_Default/learning_rate-CosineWithWarmUpLR/Add-op66,Default/learning_rate-CosineWithWarmUpLR/Add-op66,kernel_graph0,{7042-7055},7055,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7038,7059,0x12c18000b800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6587,771_Default/learning_rate-CosineWithWarmUpLR/Add-op65,Default/learning_rate-CosineWithWarmUpLR/Add-op65,kernel_graph0,{7047-7057},7057,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7042,7046,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6590,772_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op19,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op19,kernel_graph0,{7044},7044,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7044,7049,0x12c180014400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6591,773_Default/learning_rate-CosineWithWarmUpLR/Add-op67,Default/learning_rate-CosineWithWarmUpLR/Add-op67,kernel_graph0,{7047},7047,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7047,7052,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6591,774_Default/optimizer-FP32StateAdamWeightDecay/Div-op19,Default/optimizer-FP32StateAdamWeightDecay/Div-op19,kernel_graph0,{7050},7050,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7050,7055,0x12c180014400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6594,775_Default/optimizer-FP32StateAdamWeightDecay/Mul-op99,Default/optimizer-FP32StateAdamWeightDecay/Mul-op99,kernel_graph0,{7053},7053,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7053,7061,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6595,776_Default/optimizer-FP32StateAdamWeightDecay/Sub-op19,Default/optimizer-FP32StateAdamWeightDecay/Sub-op19,kernel_graph0,{7059},7059,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7062,7068,0x12c18000b800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6599,780_Default/optimizer-FP32StateAdamWeightDecay/Mul-op102,Default/optimizer-FP32StateAdamWeightDecay/Mul-op102,kernel_graph0,{7066},7066,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7064,7072,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6600,781_Default/optimizer-FP32StateAdamWeightDecay/Mul-op100,Default/optimizer-FP32StateAdamWeightDecay/Mul-op100,kernel_graph0,{7070},7070,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7066,7094,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6600,782_Default/learning_rate-CosineWithWarmUpLR/Add-op69,Default/learning_rate-CosineWithWarmUpLR/Add-op69,kernel_graph0,{7074-7092},7092,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7070,7092,0x12c18000b800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6602,783_Default/learning_rate-CosineWithWarmUpLR/Add-op68,Default/learning_rate-CosineWithWarmUpLR/Add-op68,kernel_graph0,{7079-7090},7090,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7074,7078,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6603,784_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op20,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op20,kernel_graph0,{7076},7076,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7076,7081,0x12c180010200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6603,785_Default/learning_rate-CosineWithWarmUpLR/Add-op70,Default/learning_rate-CosineWithWarmUpLR/Add-op70,kernel_graph0,{7079},7079,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7079,7085,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6605,786_Default/optimizer-FP32StateAdamWeightDecay/Div-op20,Default/optimizer-FP32StateAdamWeightDecay/Div-op20,kernel_graph0,{7082},7082,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7082,7088,0x12c180010200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6606,787_Default/optimizer-FP32StateAdamWeightDecay/Mul-op104,Default/optimizer-FP32StateAdamWeightDecay/Mul-op104,kernel_graph0,{7086},7086,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7086,7090,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6606,788_Default/optimizer-FP32StateAdamWeightDecay/Sub-op20,Default/optimizer-FP32StateAdamWeightDecay/Sub-op20,kernel_graph0,{7088},7088,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7095,9394,0x12c180009200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,6608,792_Default/TensorMove-op2,Default/TensorMove-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7099,9392,0x12c75d35ec00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/output_actor.cc,689,Other,7097,kernel_graph_0_OutputActor,UpdateOutputDeviceAddress,,{},,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 7105,7509,0x12c75d35ee00,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Kernel,7104,0_Default/network-LlamaForCausalLM/StridedSlice-op0,inputs0,0_Default/network-LlamaForCausalLM/StridedSlice-op0,{7106-7507},7507,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,1,0, 7106,8088,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7104,0_Default/network-LlamaForCausalLM/StridedSlice-op0,inputs0,0_Default/network-LlamaForCausalLM/StridedSlice-op0,{7108-7110-8085},8085,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7108,7114,0x12c18000b800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7106,1_Default/network-LlamaForCausalLM/model-LlamaModel/tok_embeddings-LlamaEmbedding/Gather-op0,Default/network-LlamaForCausalLM/model-LlamaModel/tok_embeddings-LlamaEmbedding/Gather-op0,kernel_graph0,{7112},7112,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7110,7120,0x12c18800ba00,0,DefaultEnhancedAscendMemoryPool,8704,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7107,2_Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/NotEqual-op0,Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/NotEqual-op0,kernel_graph0,{7115-7118},7118,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7112,8066,0x12c18800dc00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7107,3_Default/network-LlamaForCausalLM/model-LlamaModel/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/Cast-op0,kernel_graph0,{7124-7253-8064},8064,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7115,7543,0x12c18000b800,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7107,4_Default/network-LlamaForCausalLM/Cast-op1,Default/network-LlamaForCausalLM/Cast-op1,kernel_graph0,{7116-7135-7496-7540},7540,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7118,7149,0x12c180013a00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7108,6_Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Cast-op0,kernel_graph0,{7143-7147},7147,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7121,7992,0x12c180017c00,0,DefaultEnhancedAscendMemoryPool,524800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7108,7_Default/network-LlamaForCausalLM/model-LlamaModel/freqs_mgr-FreqsMgr/StridedSlice-op0,Default/network-LlamaForCausalLM/model-LlamaModel/freqs_mgr-FreqsMgr/StridedSlice-op0,kernel_graph0,{7122-7186-7209-7339-7354-7727-7748-7969-7990},7990,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7124,7130,0x12c18c00de00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7109,9_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op0,kernel_graph0,{7128},7128,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7126,7134,0x12c19400e000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7109,10_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{7132},7132,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7127,8073,0x12c180097e00,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7109,10_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{8069},8069,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7128,7131,0x12c1800a0000,0,DefaultEnhancedAscendMemoryPool,16777728,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7109,10_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7132,8044,0x12c1800a0000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7110,11_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op1,kernel_graph0,{7157-7162-7177-7200-8014-8028-8042},8042,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7135,7143,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7111,12_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/ReduceSum-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/ReduceSum-op0,kernel_graph0,{7141},7141,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7137,7139,0x12c180008800,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,7137,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7138,7140,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,7138,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7141,7554,0x12c180008800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7111,13_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Add-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Add-op0,kernel_graph0,{7484-7551},7551,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7145,7150,0x12c1840a0200,0,DefaultEnhancedAscendMemoryPool,2097664,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7113,15_Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/ExpandDims-op0,Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/ExpandDims-op0,kernel_graph0,{7147},7147,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7147,7153,0x12c1842a0400,0,DefaultEnhancedAscendMemoryPool,16777728,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7113,16_Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Mul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Mul-op0,kernel_graph0,{7151},7151,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7151,7157,0x12c1852a0600,0,DefaultEnhancedAscendMemoryPool,16777728,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7115,17_Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Sub-op0,Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Sub-op0,kernel_graph0,{7153-7155},7155,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7155,7950,0x12c1840a0200,0,DefaultEnhancedAscendMemoryPool,8389120,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7116,19_Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Cast-op1,kernel_graph0,{7238-7385-7703-7944},7944,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7159,8038,0x12c1848a0400,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7117,21_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/Cast-op0,kernel_graph0,{7162-8035},8035,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7161,7169,0x12c18c00de00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7118,22_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,kernel_graph0,{7164-7167},7167,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7162,7164,0x12c1868a0600,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7118,22_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7166,7173,0x12c19000e000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7120,24_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,kernel_graph0,{7171},7171,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7167,7170,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7120,24_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7171,7190,0x12c19400e200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7121,25_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op2,kernel_graph0,{7186-7188},7188,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7174,8053,0x12c18c00de00,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7122,26_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/Cast-op0,kernel_graph0,{7177-8050},8050,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7176,7184,0x12c18e00e000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7122,27_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,kernel_graph0,{7179-7182},7182,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7177,7179,0x12c1868a0600,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7122,27_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7181,7193,0x12c19c00e400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7122,29_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,kernel_graph0,{7191},7191,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7182,7185,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7122,29_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7186,7229,0x12c1a000e600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7123,30_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op2,kernel_graph0,{7227},7227,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7188,7226,0x12c1a800e800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7124,31_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op1,kernel_graph0,{7224},7224,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7191,7213,0x12c18e00e000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7125,32_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op0,kernel_graph0,{7209-7211},7211,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7194,7995,0x12c1868a0600,0,DefaultEnhancedAscendMemoryPool,524800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7125,33_Default/network-LlamaForCausalLM/model-LlamaModel/freqs_mgr-FreqsMgr/StridedSlice-op1,Default/network-LlamaForCausalLM/model-LlamaModel/freqs_mgr-FreqsMgr/StridedSlice-op1,kernel_graph0,{7195-7214-7224-7344-7359-7729-7750-7971-7993},7993,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7197,8024,0x12c19600e200,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7129,35_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/Cast-op0,kernel_graph0,{7200-8021},8021,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7199,7207,0x12c19800e400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7129,36_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,kernel_graph0,{7202-7205},7205,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7200,7202,0x12c186920800,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7129,36_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7204,7948,0x12c1b000ea00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7132,38_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,kernel_graph0,{7238-7944},7944,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7205,7208,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7132,38_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7209,7219,0x12c19800e400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7133,39_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op0,kernel_graph0,{7217},7217,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7211,7216,0x12c1b400ec00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7133,40_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op0,kernel_graph0,{7214},7214,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7214,7220,0x12c18e00e000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7135,41_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op1,kernel_graph0,{7217},7217,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7217,7223,0x12c1b400ec00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7136,42_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op0,kernel_graph0,{7221},7221,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7221,7946,0x12c18e00e000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7136,43_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op1,kernel_graph0,{7238-7944},7944,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7224,7230,0x12c19800e400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7136,44_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op3,kernel_graph0,{7227},7227,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7227,7233,0x12c1a800e800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7136,45_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op1,kernel_graph0,{7231},7231,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7231,7947,0x12c19800e400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7137,46_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op3,kernel_graph0,{7238-7944},7944,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7234,7951,0x12c186920800,0,DefaultEnhancedAscendMemoryPool,8389120,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7138,47_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{7944},7944,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7235,7952,0x12c187120a00,0,DefaultEnhancedAscendMemoryPool,8389120,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7138,47_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{7944},7944,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7236,7953,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7138,47_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{7944},7944,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7237,7954,0x12c19c00e600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7138,47_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{7242-7944},7944,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7238,7240,0x12c1a000e800,0,DefaultEnhancedAscendMemoryPool,84936192,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7138,47_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7241,7924,0x12c1a000e800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7138,48_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,kernel_graph0,{7244-7249-7922},7922,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7242,7244,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7138,48_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7246,7932,0x12c19200e200,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7141,50_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/Cast-op0,kernel_graph0,{7249-7930},7930,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7248,7255,0x12c1a400ea00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7142,51_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,kernel_graph0,{7251-7253},7253,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7249,7251,0x12c19400e400,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7142,51_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7253,7904,0x12c1a800ec00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7143,53_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/Add-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/Add-op0,kernel_graph0,{7256-7294-7902},7902,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7256,7262,0x12c1b400ec00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7144,54_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op0,kernel_graph0,{7260},7260,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7258,7266,0x12c1bc00ee00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7145,55_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{7264},7264,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7259,7911,0x12c187920c00,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7145,55_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{7907},7907,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7260,7263,0x12c19400e400,0,DefaultEnhancedAscendMemoryPool,16777728,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7145,55_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7264,7883,0x12c1a400ea00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7146,56_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op1,kernel_graph0,{7266-7271-7278-7867-7881},7881,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7268,7877,0x12c1b400ec00,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7147,58_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/Cast-op0,kernel_graph0,{7271-7874},7874,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7270,7859,0x12c1b960ee00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7148,59_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,kernel_graph0,{7273-7284-7857},7857,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7271,7273,0x12c19400e400,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7148,59_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7275,7892,0x12c1c420f000,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7151,61_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/Cast-op0,kernel_graph0,{7278-7889},7889,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7277,7865,0x12c1c980f200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7152,62_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,kernel_graph0,{7281-7862},7862,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7278,7280,0x12c19400e400,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7152,62_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7281,7855,0x12c1d440f400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7153,63_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/SiLU-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/SiLU-op0,kernel_graph0,{7282-7284-7853},7853,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7284,7842,0x12c1df00f600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7155,65_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/Mul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/Mul-op0,kernel_graph0,{7285-7290-7840},7840,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7287,7850,0x12c1e9c0f800,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7156,67_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/Cast-op0,kernel_graph0,{7290-7848},7848,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7289,7296,0x12c1ef20fa00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7157,68_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,kernel_graph0,{7292-7294},7294,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7290,7292,0x12c19400e400,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7157,68_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7294,7822,0x12c1f320fc00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7159,70_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/Add-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/Add-op1,kernel_graph0,{7297-7400-7820},7820,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7297,7303,0x12c1f720fe00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7160,71_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op0,kernel_graph0,{7301},7301,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7299,7307,0x12c1ff210000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7161,72_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{7305},7305,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7300,7829,0x12c187928e00,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7161,72_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{7825},7825,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7301,7304,0x12c19400e400,0,DefaultEnhancedAscendMemoryPool,16777728,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7161,72_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7305,7800,0x12c1ef20fa00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7162,73_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op1,kernel_graph0,{7307-7312-7327-7372-7770-7784-7798},7798,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7309,7794,0x12c1ac00ee00,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7163,75_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/Cast-op0,kernel_graph0,{7312-7791},7791,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7311,7319,0x12c1f720fe00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7164,76_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,kernel_graph0,{7314-7317},7317,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7312,7314,0x12c1ae00f000,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7164,76_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7316,7323,0x12c1fb210000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7167,78_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,kernel_graph0,{7321},7321,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7317,7320,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7167,78_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7321,7343,0x12c1ff210200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7168,79_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op2,kernel_graph0,{7339-7341},7341,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7324,7809,0x12c1f720fe00,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7168,80_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/Cast-op0,kernel_graph0,{7327-7806},7806,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7326,7334,0x12c1f9210000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7171,81_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,kernel_graph0,{7329-7332},7332,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7327,7329,0x12c1ae00f000,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7171,81_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7331,7338,0x12c207210400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7172,83_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,kernel_graph0,{7336},7336,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7332,7335,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7172,83_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7336,7358,0x12c20b210600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7174,84_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op0,kernel_graph0,{7354-7356},7356,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7339,7349,0x12c213210800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7175,85_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op2,kernel_graph0,{7347},7347,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7341,7346,0x12c21b210a00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7175,86_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op1,kernel_graph0,{7344},7344,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7344,7350,0x12c1f9210000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7177,87_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op3,kernel_graph0,{7347},7347,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7347,7353,0x12c201210200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7178,88_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op1,kernel_graph0,{7351},7351,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7351,7706,0x12c1f9210000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7178,89_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op3,kernel_graph0,{7385-7703},7703,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7354,7364,0x12c1fd210200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7179,90_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op0,kernel_graph0,{7362},7362,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7356,7361,0x12c213210800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7180,91_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op0,kernel_graph0,{7359},7359,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7359,7365,0x12c205210400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7182,92_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op1,kernel_graph0,{7362},7362,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7362,7368,0x12c20d210600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7183,93_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op0,kernel_graph0,{7366},7366,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7366,7705,0x12c1fd210200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7183,94_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op1,kernel_graph0,{7385-7703},7703,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7369,7780,0x12c201210400,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7186,95_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/Cast-op0,kernel_graph0,{7372-7777},7777,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7371,7379,0x12c203210600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7187,96_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,kernel_graph0,{7374-7377},7377,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7372,7374,0x12c1ae00f000,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7187,96_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7376,7707,0x12c207210800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7188,98_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,kernel_graph0,{7385-7703},7703,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7377,7380,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7188,98_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7381,7709,0x12c1ae00f000,0,DefaultEnhancedAscendMemoryPool,8389120,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7189,99_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{7703},7703,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7382,7710,0x12c1ae80f200,0,DefaultEnhancedAscendMemoryPool,8389120,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7189,99_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{7703},7703,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7383,7711,0x12c180008a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7189,99_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{7703},7703,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7384,7712,0x12c203210600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7189,99_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{7389-7703},7703,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7385,7387,0x12c20b210a00,0,DefaultEnhancedAscendMemoryPool,84936192,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7189,99_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7388,7683,0x12c20b210a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7189,100_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,kernel_graph0,{7391-7396-7681},7681,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7389,7391,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7189,100_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7393,7691,0x12c20f210c00,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7191,102_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/Cast-op0,kernel_graph0,{7396-7689},7689,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7395,7402,0x12c211210e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7192,103_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,kernel_graph0,{7398-7400},7400,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7396,7398,0x12c19400e400,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7192,103_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7400,7658,0x12c215211000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7194,105_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/Add-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/Add-op0,kernel_graph0,{7403-7441-7656},7656,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7403,7409,0x12c219211200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7195,106_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op0,kernel_graph0,{7407},7407,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7405,7413,0x12c221211400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7195,107_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{7411},7411,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7406,7665,0x12c187931000,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7195,107_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{7661},7661,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7407,7410,0x12c19400e400,0,DefaultEnhancedAscendMemoryPool,16777728,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7195,107_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7411,7637,0x12c211210e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7195,108_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op1,kernel_graph0,{7413-7418-7425-7621-7635},7635,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7415,7631,0x12c219211200,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7195,110_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/Cast-op0,kernel_graph0,{7418-7628},7628,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7417,7613,0x12c21e811400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7196,111_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,kernel_graph0,{7420-7431-7611},7611,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7418,7420,0x12c19400e400,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7196,111_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7422,7646,0x12c229411600,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7198,113_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/Cast-op0,kernel_graph0,{7425-7643},7643,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7424,7619,0x12c22ea11800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7198,114_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,kernel_graph0,{7428-7616},7616,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7425,7427,0x12c19400e400,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7198,114_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7428,7609,0x12c239611a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7200,115_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/SiLU-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/SiLU-op0,kernel_graph0,{7429-7431-7607},7607,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7431,7596,0x12c244211c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7201,117_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/Mul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/Mul-op0,kernel_graph0,{7432-7437-7594},7594,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7434,7604,0x12c24ee11e00,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7203,119_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/Cast-op0,kernel_graph0,{7437-7602},7602,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7436,7443,0x12c254412000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7205,120_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,kernel_graph0,{7439-7441},7441,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7437,7439,0x12c19400e400,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7205,120_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7441,7575,0x12c258412200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7208,122_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/Add-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/Add-op1,kernel_graph0,{7444-7573},7573,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7444,7450,0x12c25c412400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7209,123_Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Cast-op0,kernel_graph0,{7448},7448,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7446,7454,0x12c264412600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7210,124_Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{7452},7452,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7447,7582,0x12c187939200,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7210,124_Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{7578},7578,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7448,7451,0x12c19400e400,0,DefaultEnhancedAscendMemoryPool,16777728,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7210,124_Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7452,7558,0x12c254412000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7211,125_Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Cast-op1,kernel_graph0,{7454-7459-7556},7556,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7456,7567,0x12c25c412400,0,DefaultEnhancedAscendMemoryPool,262144512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7212,127_Default/network-LlamaForCausalLM/lm_head-Linear/Cast-op0,Default/network-LlamaForCausalLM/lm_head-Linear/Cast-op0,kernel_graph0,{7459-7564},7564,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7458,7464,0x12c26be12600,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7214,128_Default/network-LlamaForCausalLM/lm_head-Linear/MatMul-op0,Default/network-LlamaForCausalLM/lm_head-Linear/MatMul-op0,kernel_graph0,{7462},7462,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7459,7461,0x12c19400e400,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7214,128_Default/network-LlamaForCausalLM/lm_head-Linear/MatMul-op0,Default/network-LlamaForCausalLM/lm_head-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7462,7471,0x12c28b212800,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7215,129_Default/network-LlamaForCausalLM/Cast-op0,Default/network-LlamaForCausalLM/Cast-op0,kernel_graph0,{7466-7469},7469,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7465,7472,0x12c187941400,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7215,130_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/ReduceMax-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/ReduceMax-op0,kernel_graph0,{7469},7469,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7466,7468,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7215,130_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/ReduceMax-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/ReduceMax-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7469,7505,0x12c2c9a12a00,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7217,131_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Sub-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Sub-op0,kernel_graph0,{7475-7503},7503,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7473,7486,0x12c180008c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7218,132_Default/Mul-op0,Default/Mul-op0,kernel_graph0,{7484},7484,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7475,7483,0x12c26be12600,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7218,133_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Exp-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Exp-op0,kernel_graph0,{7477},7477,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7477,7502,0x12c187941400,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7221,134_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/ReduceSum-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/ReduceSum-op0,kernel_graph0,{7500},7500,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7479,7481,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,7479,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7480,7482,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,7480,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7484,7490,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7222,135_Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_RealDiv/RealDiv-op0,Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_RealDiv/RealDiv-op0,kernel_graph0,{7488},7488,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7487,7495,0x12c180008c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7222,136_Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_RealDiv/SumExt-op0,Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_RealDiv/SumExt-op0,kernel_graph0,{7491-7493},7493,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7488,7491,0x12c180009600,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7222,136_Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_RealDiv/SumExt-op0,Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_RealDiv/SumExt-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7493,7498,0x12c187949600,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7224,138_Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_ReduceSum/Tile-op0,Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_ReduceSum/Tile-op0,kernel_graph0,{7496},7496,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7496,7522,0x12c187951800,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7225,139_Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_Mul/Mul-op0,Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_Mul/Mul-op0,kernel_graph0,{7498-7519},7519,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7500,7506,0x12c187949600,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7227,141_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Log-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Log-op0,kernel_graph0,{7503},7503,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7503,7528,0x12c26be12600,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7228,142_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Sub-op1,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Sub-op1,kernel_graph0,{7514-7526},7526,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7507,7513,0x12c187941400,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7230,143_Default/network-LlamaForCausalLM/StridedSlice-op1,Default/network-LlamaForCausalLM/StridedSlice-op1,kernel_graph0,{7509-7511},7511,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7511,7529,0x12c2aa612800,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7232,145_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/OneHot-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/OneHot-op0,kernel_graph0,{7516-7526},7526,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7514,7518,0x12c2e8e12a00,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7233,146_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Exp-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Exp-op0,kernel_graph0,{7516},7516,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7516,7521,0x12c327612c00,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7238,147_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Sub-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Sub-op0,kernel_graph0,{7519},7519,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7519,7525,0x12c2e8e12a00,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7239,148_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Mul-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Mul-op0,kernel_graph0,{7523},7523,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7523,7566,0x12c327612c00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7239,149_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op0,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op0,kernel_graph0,{7556-7564},7564,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7526,7532,0x12c2e8e12a00,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7239,150_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Mul-op1,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Mul-op1,kernel_graph0,{7530},7530,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7530,7539,0x12c26be12600,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7241,151_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Neg-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Neg-op0,kernel_graph0,{7533},7533,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7533,7542,0x12c187941400,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7242,152_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/ReduceSum-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/ReduceSum-op0,kernel_graph0,{7540},7540,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7535,7537,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,7535,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7536,7538,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,7536,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7540,7550,0x12c187949600,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7243,153_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Mul-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Mul-op0,kernel_graph0,{7544},7544,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7544,7553,0x12c180008c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7243,154_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/ReduceSum-op1,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/ReduceSum-op1,kernel_graph0,{7551},7551,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7546,7548,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,7546,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7547,7549,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,7547,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7551,9387,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7244,155_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/RealDiv-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/RealDiv-op0,kernel_graph0,{9385},9385,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7555,7562,0x12c346a12e00,0,DefaultEnhancedAscendMemoryPool,262144512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7245,156_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op29,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op29,kernel_graph0,{7560},7560,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7556,7559,0x12c19400e400,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7245,156_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op29,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op29,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7560,8141,0x12c26be12600,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7246,157_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op29,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op29,kernel_graph0,{8139},8139,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7563,7572,0x12c254412000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7247,158_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op0,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op0,kernel_graph0,{7568-7570},7570,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7564,7568,0x12c346a12e00,0,DefaultEnhancedAscendMemoryPool,155190272,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7247,158_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op0,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7570,7580,0x12c25c412400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7249,160_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op1,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op1,kernel_graph0,{7578},7578,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7573,7581,0x12c28b212800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7250,161_Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Cast-op2,kernel_graph0,{7578},7578,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7576,7586,0x12c254412000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7250,162_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op0,kernel_graph0,{7584},7584,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7577,8156,0x12c18000b800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7250,162_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op0,kernel_graph0,{8154},8154,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7578,7583,0x12c19400e400,0,DefaultEnhancedAscendMemoryPool,16778752,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7250,162_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7584,7673,0x12c25c412200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7251,163_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op0,kernel_graph0,{7588-7594-7602-7670},7670,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7587,7592,0x12c180008800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7252,164_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Mul-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Mul-op0,kernel_graph0,{7590},7590,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7590,8356,0x12c180008c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7254,166_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Maximum-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Maximum-op0,kernel_graph0,{8354},8354,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7593,7600,0x12c254412000,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7254,167_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op27,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op27,kernel_graph0,{7598},7598,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7594,7597,0x12c19400e400,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7254,167_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op27,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op27,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7598,8135,0x12c244211c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7256,168_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op27,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op27,kernel_graph0,{8133},8133,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7601,7614,0x12c260412400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7257,169_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op1,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op1,kernel_graph0,{7605-7607-7611},7611,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7602,7605,0x12c19400e400,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7257,169_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op1,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7607,7630,0x12c24ee11e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7260,171_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op0,kernel_graph0,{7609-7621-7628},7628,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7611,7618,0x12c239611a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7261,173_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op1,kernel_graph0,{7614-7616},7616,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7616,7645,0x12c21e811400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7264,175_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/Grad_SiLU/SiLUGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/Grad_SiLU/SiLUGrad-op0,kernel_graph0,{7635-7643},7643,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7620,7626,0x12c260412400,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7265,176_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op28,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op28,kernel_graph0,{7624},7624,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7621,7623,0x12c19400e400,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7265,176_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op28,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op28,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7624,8138,0x12c22ea11800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7266,177_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op28,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op28,kernel_graph0,{8136},8136,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7627,7651,0x12c239611a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7267,178_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op2,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op2,kernel_graph0,{7632-7649},7649,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7628,7632,0x12c19400e400,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7267,178_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op2,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7634,7641,0x12c219211200,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7269,180_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op26,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op26,kernel_graph0,{7639},7639,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7635,7638,0x12c19400e400,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7269,180_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op26,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op26,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7639,8132,0x12c260412400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7269,181_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op26,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op26,kernel_graph0,{8130},8130,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7642,7652,0x12c211210e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7271,182_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op3,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op3,kernel_graph0,{7647-7649},7649,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7643,7647,0x12c19400e400,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7271,182_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op3,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op3,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7649,7655,0x12c23d611c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7272,184_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/AddN-op0,kernel_graph0,{7653},7653,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7653,7663,0x12c24ee11e00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7273,185_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op2,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op2,kernel_graph0,{7661},7661,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7656,7664,0x12c239611a00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7274,186_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op2,kernel_graph0,{7661},7661,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7659,7669,0x12c211210e00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7275,187_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op1,kernel_graph0,{7667},7667,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7660,8150,0x12c18000fa00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7275,187_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op1,kernel_graph0,{8148},8148,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7661,7666,0x12c19400e400,0,DefaultEnhancedAscendMemoryPool,16778752,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7275,187_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7667,7672,0x12c239611a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7276,188_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op1,kernel_graph0,{7670},7670,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7670,7837,0x12c23d611c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7276,189_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/AddN-op0,kernel_graph0,{7675-7681-7689-7834},7834,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7674,7679,0x12c180008800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7278,190_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Sub-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Sub-op0,kernel_graph0,{7677},7677,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7677,8379,0x12c180009600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7279,192_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/LessEqual-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/LessEqual-op0,kernel_graph0,{8359-8377},8377,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7680,7687,0x12c241611e00,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7281,193_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op25,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op25,kernel_graph0,{7685},7685,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7681,7684,0x12c19400e400,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7281,193_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op25,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op25,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7685,8129,0x12c20b210a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7282,194_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op25,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op25,kernel_graph0,{8127},8127,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7688,7697,0x12c239611a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7282,195_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op4,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op4,kernel_graph0,{7692-7695},7695,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7689,7692,0x12c19400e400,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7282,195_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op4,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op4,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7694,7708,0x12c24ee11e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7283,197_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op0,kernel_graph0,{7703},7703,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7695,7698,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7283,197_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7699,7720,0x12c239611a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7284,198_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,kernel_graph0,{7718},7718,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7700,7717,0x12c252e12000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7284,198_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,kernel_graph0,{7715},7715,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7701,7724,0x12c256e12200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7284,198_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,kernel_graph0,{7722},7722,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7702,7713,0x12c180008800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7284,198_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7703,7714,0x12c20f210c00,0,DefaultEnhancedAscendMemoryPool,515969024,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7284,198_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7715,7731,0x12c1f9210000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7285,199_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op5,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op5,kernel_graph0,{7727-7729},7729,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7718,7752,0x12c203210600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7285,200_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op3,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op3,kernel_graph0,{7748-7750},7750,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7721,7779,0x12c239611a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7286,201_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op3,kernel_graph0,{7725-7770-7777},7777,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7722,7725,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7286,201_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op3,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7727,7737,0x12c24ee11e00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7288,203_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op4,kernel_graph0,{7735},7735,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7729,7734,0x12c256e12000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7288,204_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op5,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op5,kernel_graph0,{7732},7732,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7732,7738,0x12c1f9210000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7290,205_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op1,kernel_graph0,{7735},7735,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7735,7741,0x12c256e12000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7291,206_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op1,kernel_graph0,{7739},7739,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7739,7745,0x12c24ee11e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7291,207_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op6,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op6,kernel_graph0,{7743},7743,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7742,7793,0x12c1f9210000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7292,208_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op2,kernel_graph0,{7746-7784-7791},7791,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7743,7746,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7292,208_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7748,7758,0x12c24ee11e00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7294,210_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op2,kernel_graph0,{7756},7756,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7750,7755,0x12c256e12000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7295,211_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op3,kernel_graph0,{7753},7753,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7753,7759,0x12c203210600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7295,212_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op0,kernel_graph0,{7756},7756,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7756,7762,0x12c256e12000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7297,213_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op0,kernel_graph0,{7760},7760,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7760,7766,0x12c1fd210200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7298,214_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op4,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op4,kernel_graph0,{7764},7764,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7763,7808,0x12c203210600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7298,215_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op1,kernel_graph0,{7767-7798-7806},7806,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7764,7767,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7298,215_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7769,7775,0x12c241611e00,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7302,217_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op24,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op24,kernel_graph0,{7773},7773,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7770,7772,0x12c1ae00f000,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7302,217_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op24,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op24,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7773,8126,0x12c1fd210200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7302,218_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op24,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op24,kernel_graph0,{8124},8124,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7776,7816,0x12c207210800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7305,219_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op7,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op7,kernel_graph0,{7781-7812},7812,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7777,7781,0x12c1ae00f000,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7305,219_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op7,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op7,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7783,7789,0x12c201210400,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7306,221_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op23,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op23,kernel_graph0,{7787},7787,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7784,7786,0x12c1ae00f000,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7306,221_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op23,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op23,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7787,8123,0x12c239611a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7307,222_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op23,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op23,kernel_graph0,{8121},8121,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7790,7815,0x12c24ee11e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7308,223_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op6,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op6,kernel_graph0,{7795-7812},7812,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7791,7795,0x12c1ae00f000,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7308,223_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op6,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op6,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7797,7804,0x12c201210400,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7310,225_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op22,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op22,kernel_graph0,{7802},7802,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7798,7801,0x12c19400e400,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7310,225_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op22,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op22,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7802,8120,0x12c1ef20fa00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7310,226_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op22,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op22,kernel_graph0,{8118},8118,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7805,7814,0x12c1f9210000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7312,227_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op5,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op5,kernel_graph0,{7810-7812},7812,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7806,7810,0x12c19400e400,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7312,227_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op5,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op5,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7812,7819,0x12c201210400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7314,229_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/AddN-op0,kernel_graph0,{7817},7817,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7817,7827,0x12c24ee11e00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7314,230_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op7,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op7,kernel_graph0,{7825},7825,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7820,7828,0x12c256e12000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7315,231_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op2,kernel_graph0,{7825},7825,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7823,7833,0x12c1f320fc00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7317,232_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op2,kernel_graph0,{7831},7831,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7824,8153,0x12c187931000,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7317,232_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op2,kernel_graph0,{8151},8151,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7825,7830,0x12c25ee12200,0,DefaultEnhancedAscendMemoryPool,16778752,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7317,232_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7831,7836,0x12c201210400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7318,233_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op2,kernel_graph0,{7834},7834,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7834,7919,0x12c205210600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7318,234_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/AddN-op0,kernel_graph0,{7837-7840-7848-7916},7916,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7839,7846,0x12c23d611c00,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7322,236_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op20,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op20,kernel_graph0,{7844},7844,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7840,7843,0x12c242c11e00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7322,236_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op20,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op20,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7844,8114,0x12c1df00f600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7322,237_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op20,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op20,kernel_graph0,{8112},8112,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7847,7860,0x12c24ee11e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7324,238_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op8,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op8,kernel_graph0,{7851-7853-7857},7857,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7848,7851,0x12c19400e400,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7324,238_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op8,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op8,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7853,7876,0x12c20f210c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7325,240_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op6,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op6,kernel_graph0,{7855-7867-7874},7874,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7857,7864,0x12c1d440f400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7328,242_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op7,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op7,kernel_graph0,{7860-7862},7862,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7862,7891,0x12c1b960ee00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7329,244_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/Grad_SiLU/SiLUGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/Grad_SiLU/SiLUGrad-op1,kernel_graph0,{7881-7889},7889,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7866,7872,0x12c1e9c0f800,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7330,245_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op21,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op21,kernel_graph0,{7870},7870,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7867,7869,0x12c19400e400,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7330,245_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op21,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op21,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7870,8117,0x12c24ee11e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7332,246_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op21,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op21,kernel_graph0,{8115},8115,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7873,7897,0x12c201210400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7333,247_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op9,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op9,kernel_graph0,{7878-7895},7895,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7874,7878,0x12c19400e400,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7333,247_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op9,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op9,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7880,7887,0x12c1b400ec00,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7336,249_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op19,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op19,kernel_graph0,{7885},7885,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7881,7884,0x12c19400e400,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7336,249_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op19,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op19,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7885,8111,0x12c1c980f200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7337,250_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op19,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op19,kernel_graph0,{8109},8109,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7888,7898,0x12c1a400ea00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7337,251_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op10,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op10,kernel_graph0,{7893-7895},7895,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7889,7893,0x12c19400e400,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7337,251_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op10,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op10,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7895,7901,0x12c1e9c0f800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7340,253_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/AddN-op0,kernel_graph0,{7899},7899,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7899,7909,0x12c1f320fc00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7340,254_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op8,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op8,kernel_graph0,{7907},7907,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7902,7910,0x12c1d440f400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7341,255_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op2,kernel_graph0,{7907},7907,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7905,7915,0x12c1a400ea00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7342,256_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op3,kernel_graph0,{7913},7913,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7906,8144,0x12c187928e00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7342,256_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op3,kernel_graph0,{8142},8142,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7907,7912,0x12c19400e400,0,DefaultEnhancedAscendMemoryPool,16778752,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7342,256_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op3,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7913,7918,0x12c201210400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7342,257_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op3,kernel_graph0,{7916},7916,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7916,8081,0x12c1e9c0f800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7344,258_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/AddN-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/AddN-op1,kernel_graph0,{7919-7922-7930-8078},8078,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7921,7928,0x12c259a12000,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7345,260_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op18,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op18,kernel_graph0,{7926},7926,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7922,7925,0x12c1edc0fa00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7345,260_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op18,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op18,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7926,8108,0x12c25ba12200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7347,261_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op18,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op18,kernel_graph0,{8106},8106,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7929,7938,0x12c23d611c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7348,262_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op11,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op11,kernel_graph0,{7933-7936},7936,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7930,7933,0x12c1edc0fa00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7348,262_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op11,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op11,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7935,7949,0x12c1f320fc00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7351,264_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op4,kernel_graph0,{7944},7944,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7936,7939,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7351,264_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op4,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7940,7962,0x12c1f720fe00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7352,265_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,kernel_graph0,{7960},7960,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7941,7959,0x12c23d611c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7352,265_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,kernel_graph0,{7957},7957,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7942,7966,0x12c201210400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7352,265_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,kernel_graph0,{7964},7964,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7943,7955,0x12c180008800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7352,265_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7944,7956,0x12c20f210c00,0,DefaultEnhancedAscendMemoryPool,515969024,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7352,265_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7957,7973,0x12c18e00e000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7352,266_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op11,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op11,kernel_graph0,{7969-7971},7971,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7960,7996,0x12c1d440f400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7354,267_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op9,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op9,kernel_graph0,{7990-7993},7993,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7963,8023,0x12c205210600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7355,268_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op7,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op7,kernel_graph0,{7967-8014-8021},8021,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7964,7967,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7355,268_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op7,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op7,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7969,7979,0x12c1f320fc00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7356,270_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op10,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op10,kernel_graph0,{7977},7977,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7971,7976,0x12c20f210c00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7357,271_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op11,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op11,kernel_graph0,{7974},7974,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7974,7980,0x12c18e00e000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7357,272_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op3,kernel_graph0,{7977},7977,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7977,7983,0x12c20f210c00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7358,273_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op1,kernel_graph0,{7981},7981,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7981,7987,0x12c201210400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7359,274_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op12,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op12,kernel_graph0,{7985},7985,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7984,8037,0x12c23d611c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7360,275_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op6,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op6,kernel_graph0,{7988-8028-8035},8035,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7985,7988,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7360,275_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op6,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op6,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7990,8002,0x12c18e00e000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7362,277_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op8,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op8,kernel_graph0,{8000},8000,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7993,7999,0x12c1f320fc00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7363,278_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op9,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op9,kernel_graph0,{7997},7997,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 7997,8003,0x12c1d440f400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7363,279_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op2,kernel_graph0,{8000},8000,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8000,8006,0x12c1f320fc00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7366,280_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op0,kernel_graph0,{8004},8004,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8004,8010,0x12c201210400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7367,281_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op10,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op10,kernel_graph0,{8008},8008,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8007,8052,0x12c18e00e000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7367,282_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op5,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op5,kernel_graph0,{8011-8042-8050},8050,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8008,8011,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7367,282_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op5,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op5,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8013,8019,0x12c209210800,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7370,284_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op17,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op17,kernel_graph0,{8017},8017,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8014,8016,0x12c1edc0fa00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7370,284_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op17,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op17,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8017,8105,0x12c201210400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7370,285_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op17,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op17,kernel_graph0,{8103},8103,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8020,8060,0x12c1f320fc00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7372,286_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op14,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op14,kernel_graph0,{8025-8056},8056,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8021,8025,0x12c1edc0fa00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7372,286_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op14,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op14,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8027,8033,0x12c259a12000,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7373,288_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op16,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op16,kernel_graph0,{8031},8031,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8028,8030,0x12c1edc0fa00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7373,288_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op16,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op16,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8031,8102,0x12c1f720fe00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7374,289_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op16,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op16,kernel_graph0,{8100},8100,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8034,8059,0x12c205210600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7375,290_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op13,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op13,kernel_graph0,{8039-8056},8056,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8035,8039,0x12c1edc0fa00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7375,290_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op13,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op13,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8041,8048,0x12c1fb210000,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7378,292_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op15,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op15,kernel_graph0,{8046},8046,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8042,8045,0x12c1edc0fa00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7378,292_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op15,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op15,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8046,8099,0x12c23d611c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7378,293_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op15,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op15,kernel_graph0,{8097},8097,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8049,8058,0x12c1800a0000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7384,294_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op12,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op12,kernel_graph0,{8054-8056},8056,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8050,8054,0x12c1edc0fa00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7384,294_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op12,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op12,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8056,8063,0x12c1d440f400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7386,296_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/AddN-op0,kernel_graph0,{8061},8061,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8061,8071,0x12c20f210c00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7386,297_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op13,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op13,kernel_graph0,{8069},8069,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8064,8072,0x12c1d440f400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7387,298_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op2,kernel_graph0,{8069},8069,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8067,8077,0x12c217210e00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7389,299_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op4,kernel_graph0,{8075},8075,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8068,8147,0x12c180013c00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7389,299_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op4,kernel_graph0,{8145},8145,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8069,8074,0x12c1edc0fa00,0,DefaultEnhancedAscendMemoryPool,16778752,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7389,299_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op4,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8075,8080,0x12c1f320fc00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7390,300_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op4,kernel_graph0,{8078},8078,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8078,8084,0x12c205210600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7390,301_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/AddN-op0,kernel_graph0,{8082},8082,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8082,8087,0x12c1d440f400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7391,302_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op14,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op14,kernel_graph0,{8085},8085,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8085,8096,0x12c20f210c00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7392,303_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/tok_embeddings-LlamaEmbedding/Grad_Gather/UnsortedSegmentSum-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/tok_embeddings-LlamaEmbedding/Grad_Gather/UnsortedSegmentSum-op0,kernel_graph0,{8094},8094,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8089,8093,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7393,304_Default/Mul-op1,Default/Mul-op1,kernel_graph0,{8091},8091,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8091,8157,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7394,305_Default/Reciprocal-op0,Default/Reciprocal-op0,kernel_graph0,{8094-8097-8100-8103-8106-8109-8112-8115-8118-8121-8124-8127-8130-8133-8136-8139-8142-8145-8148-8151-8154},8154,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8094,8392,0x12c187935200,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7394,306_Default/Mul-op2,Default/Mul-op2,kernel_graph0,{8158-8318-8390},8390,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8097,8395,0x12c1f320fc00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7396,307_Default/Mul-op3,Default/Mul-op3,kernel_graph0,{8166-8318-8393},8393,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8100,8398,0x12c1e9c0f800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7397,308_Default/Mul-op4,Default/Mul-op4,kernel_graph0,{8174-8318-8396},8396,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8103,8401,0x12c1f720fe00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7397,309_Default/Mul-op5,Default/Mul-op5,kernel_graph0,{8182-8318-8399},8399,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8106,8404,0x12c23d611c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7398,310_Default/Mul-op6,Default/Mul-op6,kernel_graph0,{8190-8318-8402},8402,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8109,8407,0x12c1d440f400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7399,311_Default/Mul-op7,Default/Mul-op7,kernel_graph0,{8198-8318-8405},8405,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8112,8410,0x12c20f210c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7400,312_Default/Mul-op8,Default/Mul-op8,kernel_graph0,{8206-8318-8408},8408,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8115,8413,0x12c1df00f600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7401,313_Default/Mul-op9,Default/Mul-op9,kernel_graph0,{8214-8318-8411},8411,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8118,8416,0x12c180017e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7401,314_Default/Mul-op10,Default/Mul-op10,kernel_graph0,{8222-8318-8414},8414,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8121,8419,0x12c1edc0fa00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7403,315_Default/Mul-op11,Default/Mul-op11,kernel_graph0,{8230-8318-8417},8417,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8124,8422,0x12c239611a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7404,316_Default/Mul-op12,Default/Mul-op12,kernel_graph0,{8238-8318-8420},8420,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8127,8425,0x12c1fb210000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7404,317_Default/Mul-op13,Default/Mul-op13,kernel_graph0,{8246-8318-8423},8423,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8130,8428,0x12c1ff210200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7406,318_Default/Mul-op14,Default/Mul-op14,kernel_graph0,{8254-8318-8426},8426,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8133,8431,0x12c219e10e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7407,319_Default/Mul-op15,Default/Mul-op15,kernel_graph0,{8262-8318-8429},8429,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8136,8434,0x12c241611e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7408,320_Default/Mul-op16,Default/Mul-op16,kernel_graph0,{8270-8318-8432},8432,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8139,8437,0x12c24c212000,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7409,321_Default/Mul-op17,Default/Mul-op17,kernel_graph0,{8278-8318-8435},8435,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8142,8440,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7411,322_Default/Mul-op18,Default/Mul-op18,kernel_graph0,{8286-8318-8438},8438,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8145,8443,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7412,323_Default/Mul-op19,Default/Mul-op19,kernel_graph0,{8294-8318-8441},8441,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8148,8446,0x12c180013c00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7412,324_Default/Mul-op20,Default/Mul-op20,kernel_graph0,{8302-8318-8444},8444,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8151,8449,0x12c18000fa00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7413,325_Default/Mul-op21,Default/Mul-op21,kernel_graph0,{8310-8318-8447},8447,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8154,8452,0x12c1f1c0fc00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7414,326_Default/Mul-op22,Default/Mul-op22,kernel_graph0,{8318-8320-8450},8450,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8158,8163,0x12c1a6d35400,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7415,327_Default/clip_grad_norm-ClipGradNorm/Square-op0,Default/clip_grad_norm-ClipGradNorm/Square-op0,kernel_graph0,{8161},8161,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8160,8330,0x12c180008a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7416,328_Default/clip_grad_norm-ClipGradNorm/SumExt-op0,Default/clip_grad_norm-ClipGradNorm/SumExt-op0,kernel_graph0,{8164-8328},8328,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8161,8164,0x12c18000b800,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7416,328_Default/clip_grad_norm-ClipGradNorm/SumExt-op0,Default/clip_grad_norm-ClipGradNorm/SumExt-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8166,8171,0x12c209e10400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7418,330_Default/clip_grad_norm-ClipGradNorm/Square-op1,Default/clip_grad_norm-ClipGradNorm/Square-op1,kernel_graph0,{8169},8169,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8168,8331,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7419,331_Default/clip_grad_norm-ClipGradNorm/SumExt-op1,Default/clip_grad_norm-ClipGradNorm/SumExt-op1,kernel_graph0,{8172-8328},8328,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8169,8172,0x12c18000b800,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7419,331_Default/clip_grad_norm-ClipGradNorm/SumExt-op1,Default/clip_grad_norm-ClipGradNorm/SumExt-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8174,8179,0x12c209e10400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7420,333_Default/clip_grad_norm-ClipGradNorm/Square-op2,Default/clip_grad_norm-ClipGradNorm/Square-op2,kernel_graph0,{8177},8177,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8176,8332,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7421,334_Default/clip_grad_norm-ClipGradNorm/SumExt-op2,Default/clip_grad_norm-ClipGradNorm/SumExt-op2,kernel_graph0,{8180-8328},8328,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8177,8180,0x12c18000b800,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7421,334_Default/clip_grad_norm-ClipGradNorm/SumExt-op2,Default/clip_grad_norm-ClipGradNorm/SumExt-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8182,8187,0x12c209e10400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7423,336_Default/clip_grad_norm-ClipGradNorm/Square-op3,Default/clip_grad_norm-ClipGradNorm/Square-op3,kernel_graph0,{8185},8185,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8184,8333,0x12c180009a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7423,337_Default/clip_grad_norm-ClipGradNorm/SumExt-op3,Default/clip_grad_norm-ClipGradNorm/SumExt-op3,kernel_graph0,{8188-8328},8328,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8185,8188,0x12c18000b800,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7423,337_Default/clip_grad_norm-ClipGradNorm/SumExt-op3,Default/clip_grad_norm-ClipGradNorm/SumExt-op3,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8190,8195,0x12c209e10400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7426,339_Default/clip_grad_norm-ClipGradNorm/Square-op4,Default/clip_grad_norm-ClipGradNorm/Square-op4,kernel_graph0,{8193},8193,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8192,8334,0x12c180009c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7426,340_Default/clip_grad_norm-ClipGradNorm/SumExt-op4,Default/clip_grad_norm-ClipGradNorm/SumExt-op4,kernel_graph0,{8196-8328},8328,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8193,8196,0x12c18000b800,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7426,340_Default/clip_grad_norm-ClipGradNorm/SumExt-op4,Default/clip_grad_norm-ClipGradNorm/SumExt-op4,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8198,8203,0x12c224a11000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7428,342_Default/clip_grad_norm-ClipGradNorm/Square-op5,Default/clip_grad_norm-ClipGradNorm/Square-op5,kernel_graph0,{8201},8201,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8200,8335,0x12c180009e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7429,343_Default/clip_grad_norm-ClipGradNorm/SumExt-op5,Default/clip_grad_norm-ClipGradNorm/SumExt-op5,kernel_graph0,{8204-8328},8328,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8201,8204,0x12c18000b800,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7429,343_Default/clip_grad_norm-ClipGradNorm/SumExt-op5,Default/clip_grad_norm-ClipGradNorm/SumExt-op5,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8206,8211,0x12c224a11000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7430,345_Default/clip_grad_norm-ClipGradNorm/Square-op6,Default/clip_grad_norm-ClipGradNorm/Square-op6,kernel_graph0,{8209},8209,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8208,8336,0x12c18000a000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7431,346_Default/clip_grad_norm-ClipGradNorm/SumExt-op6,Default/clip_grad_norm-ClipGradNorm/SumExt-op6,kernel_graph0,{8212-8328},8328,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8209,8212,0x12c18000b800,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7431,346_Default/clip_grad_norm-ClipGradNorm/SumExt-op6,Default/clip_grad_norm-ClipGradNorm/SumExt-op6,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8214,8219,0x12c224a11000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7432,348_Default/clip_grad_norm-ClipGradNorm/Square-op7,Default/clip_grad_norm-ClipGradNorm/Square-op7,kernel_graph0,{8217},8217,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8216,8337,0x12c18000a200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7432,349_Default/clip_grad_norm-ClipGradNorm/SumExt-op7,Default/clip_grad_norm-ClipGradNorm/SumExt-op7,kernel_graph0,{8220-8328},8328,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8217,8220,0x12c18000b800,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7432,349_Default/clip_grad_norm-ClipGradNorm/SumExt-op7,Default/clip_grad_norm-ClipGradNorm/SumExt-op7,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8222,8227,0x12c209e10400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7434,351_Default/clip_grad_norm-ClipGradNorm/Square-op8,Default/clip_grad_norm-ClipGradNorm/Square-op8,kernel_graph0,{8225},8225,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8224,8338,0x12c18000a400,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7435,352_Default/clip_grad_norm-ClipGradNorm/SumExt-op8,Default/clip_grad_norm-ClipGradNorm/SumExt-op8,kernel_graph0,{8228-8328},8328,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8225,8228,0x12c18000b800,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7435,352_Default/clip_grad_norm-ClipGradNorm/SumExt-op8,Default/clip_grad_norm-ClipGradNorm/SumExt-op8,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8230,8235,0x12c209e10400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7437,354_Default/clip_grad_norm-ClipGradNorm/Square-op9,Default/clip_grad_norm-ClipGradNorm/Square-op9,kernel_graph0,{8233},8233,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8232,8339,0x12c18000a600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7438,355_Default/clip_grad_norm-ClipGradNorm/SumExt-op9,Default/clip_grad_norm-ClipGradNorm/SumExt-op9,kernel_graph0,{8236-8328},8328,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8233,8236,0x12c18000b800,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7438,355_Default/clip_grad_norm-ClipGradNorm/SumExt-op9,Default/clip_grad_norm-ClipGradNorm/SumExt-op9,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8238,8243,0x12c209e10400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7439,357_Default/clip_grad_norm-ClipGradNorm/Square-op10,Default/clip_grad_norm-ClipGradNorm/Square-op10,kernel_graph0,{8241},8241,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8240,8340,0x12c18000a800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7440,358_Default/clip_grad_norm-ClipGradNorm/SumExt-op10,Default/clip_grad_norm-ClipGradNorm/SumExt-op10,kernel_graph0,{8244-8328},8328,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8241,8244,0x12c18000b800,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7440,358_Default/clip_grad_norm-ClipGradNorm/SumExt-op10,Default/clip_grad_norm-ClipGradNorm/SumExt-op10,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8246,8251,0x12c209e10400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7442,360_Default/clip_grad_norm-ClipGradNorm/Square-op11,Default/clip_grad_norm-ClipGradNorm/Square-op11,kernel_graph0,{8249},8249,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8248,8341,0x12c18000aa00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7442,361_Default/clip_grad_norm-ClipGradNorm/SumExt-op11,Default/clip_grad_norm-ClipGradNorm/SumExt-op11,kernel_graph0,{8252-8328},8328,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8249,8252,0x12c18000b800,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7442,361_Default/clip_grad_norm-ClipGradNorm/SumExt-op11,Default/clip_grad_norm-ClipGradNorm/SumExt-op11,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8254,8259,0x12c224a11000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7445,363_Default/clip_grad_norm-ClipGradNorm/Square-op12,Default/clip_grad_norm-ClipGradNorm/Square-op12,kernel_graph0,{8257},8257,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8256,8342,0x12c18000ac00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7445,364_Default/clip_grad_norm-ClipGradNorm/SumExt-op12,Default/clip_grad_norm-ClipGradNorm/SumExt-op12,kernel_graph0,{8260-8328},8328,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8257,8260,0x12c18000b800,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7445,364_Default/clip_grad_norm-ClipGradNorm/SumExt-op12,Default/clip_grad_norm-ClipGradNorm/SumExt-op12,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8262,8267,0x12c224a11000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7448,366_Default/clip_grad_norm-ClipGradNorm/Square-op13,Default/clip_grad_norm-ClipGradNorm/Square-op13,kernel_graph0,{8265},8265,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8264,8343,0x12c18000ae00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7449,367_Default/clip_grad_norm-ClipGradNorm/SumExt-op13,Default/clip_grad_norm-ClipGradNorm/SumExt-op13,kernel_graph0,{8268-8328},8328,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8265,8268,0x12c18000b800,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7449,367_Default/clip_grad_norm-ClipGradNorm/SumExt-op13,Default/clip_grad_norm-ClipGradNorm/SumExt-op13,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8270,8275,0x12c224a11000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7452,369_Default/clip_grad_norm-ClipGradNorm/Square-op14,Default/clip_grad_norm-ClipGradNorm/Square-op14,kernel_graph0,{8273},8273,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8272,8344,0x12c18000b000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7453,370_Default/clip_grad_norm-ClipGradNorm/SumExt-op14,Default/clip_grad_norm-ClipGradNorm/SumExt-op14,kernel_graph0,{8276-8328},8328,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8273,8276,0x12c18000b800,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7453,370_Default/clip_grad_norm-ClipGradNorm/SumExt-op14,Default/clip_grad_norm-ClipGradNorm/SumExt-op14,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8278,8283,0x12c1a6d35400,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7454,372_Default/clip_grad_norm-ClipGradNorm/Square-op15,Default/clip_grad_norm-ClipGradNorm/Square-op15,kernel_graph0,{8281},8281,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8280,8345,0x12c18000b200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7455,373_Default/clip_grad_norm-ClipGradNorm/SumExt-op15,Default/clip_grad_norm-ClipGradNorm/SumExt-op15,kernel_graph0,{8284-8328},8328,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8281,8284,0x12c18000b800,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7455,373_Default/clip_grad_norm-ClipGradNorm/SumExt-op15,Default/clip_grad_norm-ClipGradNorm/SumExt-op15,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8286,8291,0x12c18000b800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7457,375_Default/clip_grad_norm-ClipGradNorm/Square-op16,Default/clip_grad_norm-ClipGradNorm/Square-op16,kernel_graph0,{8289},8289,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8288,8346,0x12c1f1c13e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7458,376_Default/clip_grad_norm-ClipGradNorm/SumExt-op16,Default/clip_grad_norm-ClipGradNorm/SumExt-op16,kernel_graph0,{8292-8328},8328,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8289,8292,0x12c1f1c14000,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7458,376_Default/clip_grad_norm-ClipGradNorm/SumExt-op16,Default/clip_grad_norm-ClipGradNorm/SumExt-op16,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8294,8299,0x12c18000b800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7460,378_Default/clip_grad_norm-ClipGradNorm/Square-op17,Default/clip_grad_norm-ClipGradNorm/Square-op17,kernel_graph0,{8297},8297,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8296,8347,0x12c1f1c14000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7461,379_Default/clip_grad_norm-ClipGradNorm/SumExt-op17,Default/clip_grad_norm-ClipGradNorm/SumExt-op17,kernel_graph0,{8300-8328},8328,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8297,8300,0x12c1f1c14200,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7461,379_Default/clip_grad_norm-ClipGradNorm/SumExt-op17,Default/clip_grad_norm-ClipGradNorm/SumExt-op17,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8302,8307,0x12c18000b800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7463,381_Default/clip_grad_norm-ClipGradNorm/Square-op18,Default/clip_grad_norm-ClipGradNorm/Square-op18,kernel_graph0,{8305},8305,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8304,8348,0x12c1f1c14200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7463,382_Default/clip_grad_norm-ClipGradNorm/SumExt-op18,Default/clip_grad_norm-ClipGradNorm/SumExt-op18,kernel_graph0,{8308-8328},8328,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8305,8308,0x12c1f1c14400,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7463,382_Default/clip_grad_norm-ClipGradNorm/SumExt-op18,Default/clip_grad_norm-ClipGradNorm/SumExt-op18,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8310,8315,0x12c18000b800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7467,384_Default/clip_grad_norm-ClipGradNorm/Square-op19,Default/clip_grad_norm-ClipGradNorm/Square-op19,kernel_graph0,{8313},8313,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8312,8349,0x12c1f1c14400,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7467,385_Default/clip_grad_norm-ClipGradNorm/SumExt-op19,Default/clip_grad_norm-ClipGradNorm/SumExt-op19,kernel_graph0,{8316-8328},8328,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8313,8316,0x12c1f1c14600,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7467,385_Default/clip_grad_norm-ClipGradNorm/SumExt-op19,Default/clip_grad_norm-ClipGradNorm/SumExt-op19,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8318,11683,0x12c18000b800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7470,387_Default/AllFinite-op0,Default/AllFinite-op0,kernel_graph0,{8354-8357-8359},8359,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8320,8325,0x12c1f1c14600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7470,388_Default/clip_grad_norm-ClipGradNorm/Square-op20,Default/clip_grad_norm-ClipGradNorm/Square-op20,kernel_graph0,{8323},8323,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8322,8350,0x12c18000ba00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7473,389_Default/clip_grad_norm-ClipGradNorm/SumExt-op20,Default/clip_grad_norm-ClipGradNorm/SumExt-op20,kernel_graph0,{8326-8328},8328,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8323,8326,0x12c18000bc00,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7473,389_Default/clip_grad_norm-ClipGradNorm/SumExt-op20,Default/clip_grad_norm-ClipGradNorm/SumExt-op20,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8328,8353,0x12c18000bc00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7474,391_Default/clip_grad_norm-ClipGradNorm/AddN-op0,Default/clip_grad_norm-ClipGradNorm/AddN-op0,kernel_graph0,{8351},8351,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8351,11680,0x12c180008a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7475,392_Default/clip_grad_norm-ClipGradNorm/Sqrt-op0,Default/clip_grad_norm-ClipGradNorm/Sqrt-op0,kernel_graph0,{8361-8363},8363,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8354,8384,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7476,393_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Select-op1,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Select-op1,kernel_graph0,{8375-8380},8380,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8357,8501,0x12c180008c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7476,394_Default/LogicalNot-op0,Default/LogicalNot-op0,kernel_graph0,{8377},8377,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8359,8387,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7477,395_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/LogicalOr-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/LogicalOr-op0,kernel_graph0,{8385},8385,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8361,8365,0x12c180009a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7478,396_Default/clip_grad_norm-ClipGradNorm/GreaterEqual-op0,Default/clip_grad_norm-ClipGradNorm/GreaterEqual-op0,kernel_graph0,{8363},8363,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8363,8368,0x12c180009c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7478,397_Default/clip_grad_norm-ClipGradNorm/Select-op0,Default/clip_grad_norm-ClipGradNorm/Select-op0,kernel_graph0,{8366},8366,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8366,8371,0x12c180009a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7478,398_Default/learning_rate-CosineWithWarmUpLR/Add-op0,Default/learning_rate-CosineWithWarmUpLR/Add-op0,kernel_graph0,{8369},8369,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8369,8374,0x12c180009c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7479,399_Default/clip_grad_norm-ClipGradNorm/Div-op0,Default/clip_grad_norm-ClipGradNorm/Div-op0,kernel_graph0,{8372},8372,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8372,8453,0x12c180009a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7479,400_Default/clip_grad_norm-ClipGradNorm/Minimum-op0,Default/clip_grad_norm-ClipGradNorm/Minimum-op0,kernel_graph0,{8390-8393-8396-8399-8402-8405-8408-8411-8414-8417-8420-8423-8426-8429-8432-8435-8438-8441-8444-8447-8450},8450,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8375,8383,0x12c180009c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7480,401_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Mul-op1,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Mul-op1,kernel_graph0,{8380},8380,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8377,8382,0x12c180009e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7480,402_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/LogicalAnd-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/LogicalAnd-op0,kernel_graph0,{8380},8380,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8380,8459,0x12c180009600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7484,403_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Select-op2,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Select-op2,kernel_graph0,{8457},8457,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8385,8455,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7484,404_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Select-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Select-op0,kernel_graph0,{8453},8453,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8388,8457,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7485,405_Default/learning_rate-CosineWithWarmUpLR/Add-op1,Default/learning_rate-CosineWithWarmUpLR/Add-op1,kernel_graph0,{8455},8455,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8390,8660,0x12c1a6d35400,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7485,406_Default/clip_grad_norm-ClipGradNorm/Mul-op0,Default/clip_grad_norm-ClipGradNorm/Mul-op0,kernel_graph0,{8656-8658},8658,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8393,8650,0x12c209e10400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7485,407_Default/clip_grad_norm-ClipGradNorm/Mul-op1,Default/clip_grad_norm-ClipGradNorm/Mul-op1,kernel_graph0,{8646-8648},8648,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8396,8642,0x12c1f1c13e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7486,408_Default/clip_grad_norm-ClipGradNorm/Mul-op2,Default/clip_grad_norm-ClipGradNorm/Mul-op2,kernel_graph0,{8638-8640},8640,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8399,8634,0x12c1e9c0f800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7488,409_Default/clip_grad_norm-ClipGradNorm/Mul-op3,Default/clip_grad_norm-ClipGradNorm/Mul-op3,kernel_graph0,{8630-8632},8632,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8402,8626,0x12c1f5c14000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7489,410_Default/clip_grad_norm-ClipGradNorm/Mul-op4,Default/clip_grad_norm-ClipGradNorm/Mul-op4,kernel_graph0,{8622-8624},8624,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8405,8618,0x12c1c6135600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7489,411_Default/clip_grad_norm-ClipGradNorm/Mul-op5,Default/clip_grad_norm-ClipGradNorm/Mul-op5,kernel_graph0,{8614-8616},8616,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8408,8610,0x12c1d0d35800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7491,412_Default/clip_grad_norm-ClipGradNorm/Mul-op6,Default/clip_grad_norm-ClipGradNorm/Mul-op6,kernel_graph0,{8606-8608},8608,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8411,8602,0x12c20de10600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7491,413_Default/clip_grad_norm-ClipGradNorm/Mul-op7,Default/clip_grad_norm-ClipGradNorm/Mul-op7,kernel_graph0,{8598-8600},8600,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8414,8594,0x12c23d611c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7492,414_Default/clip_grad_norm-ClipGradNorm/Mul-op8,Default/clip_grad_norm-ClipGradNorm/Mul-op8,kernel_graph0,{8590-8592},8592,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8417,8586,0x12c1db935a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7493,415_Default/clip_grad_norm-ClipGradNorm/Mul-op9,Default/clip_grad_norm-ClipGradNorm/Mul-op9,kernel_graph0,{8582-8584},8584,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8420,8578,0x12c1edc0fa00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7494,416_Default/clip_grad_norm-ClipGradNorm/Mul-op10,Default/clip_grad_norm-ClipGradNorm/Mul-op10,kernel_graph0,{8574-8576},8576,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8423,8570,0x12c1df935c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7494,417_Default/clip_grad_norm-ClipGradNorm/Mul-op11,Default/clip_grad_norm-ClipGradNorm/Mul-op11,kernel_graph0,{8566-8568},8568,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8426,8562,0x12c224a11000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7495,418_Default/clip_grad_norm-ClipGradNorm/Mul-op12,Default/clip_grad_norm-ClipGradNorm/Mul-op12,kernel_graph0,{8558-8560},8560,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8429,8554,0x12c22f611200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7496,419_Default/clip_grad_norm-ClipGradNorm/Mul-op13,Default/clip_grad_norm-ClipGradNorm/Mul-op13,kernel_graph0,{8550-8552},8552,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8432,8546,0x12c218a10800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7497,420_Default/clip_grad_norm-ClipGradNorm/Mul-op14,Default/clip_grad_norm-ClipGradNorm/Mul-op14,kernel_graph0,{8542-8544},8544,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8435,8538,0x12c180017e00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7498,421_Default/clip_grad_norm-ClipGradNorm/Mul-op15,Default/clip_grad_norm-ClipGradNorm/Mul-op15,kernel_graph0,{8534-8536},8536,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8438,8530,0x12c223610a00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7498,422_Default/clip_grad_norm-ClipGradNorm/Mul-op16,Default/clip_grad_norm-ClipGradNorm/Mul-op16,kernel_graph0,{8526-8528},8528,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8441,8522,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7499,423_Default/clip_grad_norm-ClipGradNorm/Mul-op17,Default/clip_grad_norm-ClipGradNorm/Mul-op17,kernel_graph0,{8518-8520},8520,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8444,8514,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7500,424_Default/clip_grad_norm-ClipGradNorm/Mul-op18,Default/clip_grad_norm-ClipGradNorm/Mul-op18,kernel_graph0,{8510-8512},8512,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8447,8506,0x12c180013c00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7501,425_Default/clip_grad_norm-ClipGradNorm/Mul-op19,Default/clip_grad_norm-ClipGradNorm/Mul-op19,kernel_graph0,{8502-8504},8504,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8450,8668,0x12c18000ba00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7501,426_Default/clip_grad_norm-ClipGradNorm/Mul-op20,Default/clip_grad_norm-ClipGradNorm/Mul-op20,kernel_graph0,{8664-8666},8666,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8460,8469,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7507,430_Default/learning_rate-CosineWithWarmUpLR/Cast-op0,Default/learning_rate-CosineWithWarmUpLR/Cast-op0,kernel_graph0,{8462-8465-8467},8467,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8462,8464,0x12c180009600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,7508,431_Default/learning_rate-CosineWithWarmUpLR/GreaterEqual-op0,Default/learning_rate-CosineWithWarmUpLR/GreaterEqual-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8465,8470,0x12c180009600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8464,435_Default/learning_rate-CosineWithWarmUpLR/Greater-op0,Default/learning_rate-CosineWithWarmUpLR/Greater-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8467,8473,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8465,436_Default/TensorMove-op0,Default/TensorMove-op0,kernel_graph0,{8471},8471,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8471,8476,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8470,440_Default/learning_rate-CosineWithWarmUpLR/Sub-op1,Default/learning_rate-CosineWithWarmUpLR/Sub-op1,kernel_graph0,{8474},8474,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8474,8479,0x12c180009600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8471,441_Default/learning_rate-CosineWithWarmUpLR/Div-op1,Default/learning_rate-CosineWithWarmUpLR/Div-op1,kernel_graph0,{8477},8477,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8477,8482,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8472,442_Default/learning_rate-CosineWithWarmUpLR/Mul-op1,Default/learning_rate-CosineWithWarmUpLR/Mul-op1,kernel_graph0,{8480},8480,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8480,8485,0x12c180009600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8472,443_Default/learning_rate-CosineWithWarmUpLR/Cos-op0,Default/learning_rate-CosineWithWarmUpLR/Cos-op0,kernel_graph0,{8483},8483,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8483,8488,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8474,444_Default/learning_rate-CosineWithWarmUpLR/Add-op3,Default/learning_rate-CosineWithWarmUpLR/Add-op3,kernel_graph0,{8486},8486,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8486,8491,0x12c180009600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8475,445_Default/learning_rate-CosineWithWarmUpLR/Mul-op2,Default/learning_rate-CosineWithWarmUpLR/Mul-op2,kernel_graph0,{8489},8489,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8489,8494,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8475,446_Default/learning_rate-CosineWithWarmUpLR/Maximum-op0,Default/learning_rate-CosineWithWarmUpLR/Maximum-op0,kernel_graph0,{8492},8492,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8492,8497,0x12c180009600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8477,447_Default/learning_rate-CosineWithWarmUpLR/Mul-op3,Default/learning_rate-CosineWithWarmUpLR/Mul-op3,kernel_graph0,{8495},8495,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8495,8500,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8478,448_Default/learning_rate-CosineWithWarmUpLR/Add-op4,Default/learning_rate-CosineWithWarmUpLR/Add-op4,kernel_graph0,{8498},8498,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8498,11681,0x12c180009600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8481,451_Default/Reshape-op0,Default/Reshape-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8502,9331,0x12c1f1c0fc00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8501,453_Default/optimizer-FP32StateAdamWeightDecay/Mul-op96,Default/optimizer-FP32StateAdamWeightDecay/Mul-op96,kernel_graph0,{9328},9328,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8504,8509,0x12c223614c00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8502,454_Default/optimizer-FP32StateAdamWeightDecay/Square-op19,Default/optimizer-FP32StateAdamWeightDecay/Square-op19,kernel_graph0,{8507},8507,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8507,9327,0x12c18000fc00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8503,455_Default/optimizer-FP32StateAdamWeightDecay/Mul-op98,Default/optimizer-FP32StateAdamWeightDecay/Mul-op98,kernel_graph0,{9324},9324,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8510,9295,0x12c223614c00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8503,456_Default/optimizer-FP32StateAdamWeightDecay/Mul-op91,Default/optimizer-FP32StateAdamWeightDecay/Mul-op91,kernel_graph0,{9292},9292,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8512,8517,0x12c223618e00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8504,457_Default/optimizer-FP32StateAdamWeightDecay/Square-op18,Default/optimizer-FP32StateAdamWeightDecay/Square-op18,kernel_graph0,{8515},8515,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8515,9299,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8505,458_Default/optimizer-FP32StateAdamWeightDecay/Mul-op93,Default/optimizer-FP32StateAdamWeightDecay/Mul-op93,kernel_graph0,{9296},9296,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8518,9263,0x12c223618e00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8505,459_Default/optimizer-FP32StateAdamWeightDecay/Mul-op86,Default/optimizer-FP32StateAdamWeightDecay/Mul-op86,kernel_graph0,{9260},9260,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8520,8525,0x12c22361d000,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8507,460_Default/optimizer-FP32StateAdamWeightDecay/Square-op17,Default/optimizer-FP32StateAdamWeightDecay/Square-op17,kernel_graph0,{8523},8523,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8523,9267,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8508,461_Default/optimizer-FP32StateAdamWeightDecay/Mul-op88,Default/optimizer-FP32StateAdamWeightDecay/Mul-op88,kernel_graph0,{9264},9264,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8526,9231,0x12c22361d000,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8508,462_Default/optimizer-FP32StateAdamWeightDecay/Mul-op81,Default/optimizer-FP32StateAdamWeightDecay/Mul-op81,kernel_graph0,{9228},9228,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8528,8533,0x12c223621200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8510,463_Default/optimizer-FP32StateAdamWeightDecay/Square-op16,Default/optimizer-FP32StateAdamWeightDecay/Square-op16,kernel_graph0,{8531},8531,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8531,9235,0x12c223610a00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8511,464_Default/optimizer-FP32StateAdamWeightDecay/Mul-op83,Default/optimizer-FP32StateAdamWeightDecay/Mul-op83,kernel_graph0,{9232},9232,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8534,9199,0x12c241611e00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8511,465_Default/optimizer-FP32StateAdamWeightDecay/Mul-op76,Default/optimizer-FP32StateAdamWeightDecay/Mul-op76,kernel_graph0,{9196},9196,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8536,8541,0x12c260a12000,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8512,466_Default/optimizer-FP32StateAdamWeightDecay/Square-op15,Default/optimizer-FP32StateAdamWeightDecay/Square-op15,kernel_graph0,{8539},8539,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8539,9203,0x12c180013e00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8513,467_Default/optimizer-FP32StateAdamWeightDecay/Mul-op78,Default/optimizer-FP32StateAdamWeightDecay/Mul-op78,kernel_graph0,{9200},9200,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8542,9167,0x12c1f9c14200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8513,468_Default/optimizer-FP32StateAdamWeightDecay/Mul-op71,Default/optimizer-FP32StateAdamWeightDecay/Mul-op71,kernel_graph0,{9164},9164,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8544,8549,0x12c260a12000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8515,469_Default/optimizer-FP32StateAdamWeightDecay/Square-op14,Default/optimizer-FP32StateAdamWeightDecay/Square-op14,kernel_graph0,{8547},8547,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8547,9171,0x12c218a10800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8516,470_Default/optimizer-FP32StateAdamWeightDecay/Mul-op73,Default/optimizer-FP32StateAdamWeightDecay/Mul-op73,kernel_graph0,{9168},9168,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8550,9135,0x12c260a12000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8516,471_Default/optimizer-FP32StateAdamWeightDecay/Mul-op66,Default/optimizer-FP32StateAdamWeightDecay/Mul-op66,kernel_graph0,{9132},9132,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8552,8557,0x12c26b612200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8518,472_Default/optimizer-FP32StateAdamWeightDecay/Square-op13,Default/optimizer-FP32StateAdamWeightDecay/Square-op13,kernel_graph0,{8555},8555,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8555,9139,0x12c22f611200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8519,473_Default/optimizer-FP32StateAdamWeightDecay/Mul-op68,Default/optimizer-FP32StateAdamWeightDecay/Mul-op68,kernel_graph0,{9136},9136,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8558,9103,0x12c26b612200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8519,474_Default/optimizer-FP32StateAdamWeightDecay/Mul-op61,Default/optimizer-FP32StateAdamWeightDecay/Mul-op61,kernel_graph0,{9100},9100,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8560,8565,0x12c276212400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8520,475_Default/optimizer-FP32StateAdamWeightDecay/Square-op12,Default/optimizer-FP32StateAdamWeightDecay/Square-op12,kernel_graph0,{8563},8563,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8563,9107,0x12c223621200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8521,476_Default/optimizer-FP32StateAdamWeightDecay/Mul-op63,Default/optimizer-FP32StateAdamWeightDecay/Mul-op63,kernel_graph0,{9104},9104,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8566,9071,0x12c204814400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8521,477_Default/optimizer-FP32StateAdamWeightDecay/Mul-op56,Default/optimizer-FP32StateAdamWeightDecay/Mul-op56,kernel_graph0,{9068},9068,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8568,8573,0x12c1e3935e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8523,478_Default/optimizer-FP32StateAdamWeightDecay/Square-op11,Default/optimizer-FP32StateAdamWeightDecay/Square-op11,kernel_graph0,{8571},8571,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8571,9075,0x12c1df935c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8524,479_Default/optimizer-FP32StateAdamWeightDecay/Mul-op58,Default/optimizer-FP32StateAdamWeightDecay/Mul-op58,kernel_graph0,{9072},9072,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8574,9039,0x12c1e3935e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8524,480_Default/optimizer-FP32StateAdamWeightDecay/Mul-op51,Default/optimizer-FP32StateAdamWeightDecay/Mul-op51,kernel_graph0,{9036},9036,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8576,8581,0x12c19f414000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8526,481_Default/optimizer-FP32StateAdamWeightDecay/Square-op10,Default/optimizer-FP32StateAdamWeightDecay/Square-op10,kernel_graph0,{8579},8579,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8579,9043,0x12c1edc0fa00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8526,482_Default/optimizer-FP32StateAdamWeightDecay/Mul-op53,Default/optimizer-FP32StateAdamWeightDecay/Mul-op53,kernel_graph0,{9040},9040,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8582,9007,0x12c19f414000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8527,483_Default/optimizer-FP32StateAdamWeightDecay/Mul-op46,Default/optimizer-FP32StateAdamWeightDecay/Mul-op46,kernel_graph0,{9004},9004,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8584,8589,0x12c276212400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8527,484_Default/optimizer-FP32StateAdamWeightDecay/Square-op9,Default/optimizer-FP32StateAdamWeightDecay/Square-op9,kernel_graph0,{8587},8587,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8587,9011,0x12c1db935a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8528,485_Default/optimizer-FP32StateAdamWeightDecay/Mul-op48,Default/optimizer-FP32StateAdamWeightDecay/Mul-op48,kernel_graph0,{9008},9008,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8590,8975,0x12c276212400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8529,486_Default/optimizer-FP32StateAdamWeightDecay/Mul-op41,Default/optimizer-FP32StateAdamWeightDecay/Mul-op41,kernel_graph0,{8972},8972,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8592,8597,0x12c27a212600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8530,487_Default/optimizer-FP32StateAdamWeightDecay/Square-op8,Default/optimizer-FP32StateAdamWeightDecay/Square-op8,kernel_graph0,{8595},8595,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8595,8979,0x12c23a211400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8531,488_Default/optimizer-FP32StateAdamWeightDecay/Mul-op43,Default/optimizer-FP32StateAdamWeightDecay/Mul-op43,kernel_graph0,{8976},8976,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8598,8943,0x12c27a212600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8532,489_Default/optimizer-FP32StateAdamWeightDecay/Mul-op36,Default/optimizer-FP32StateAdamWeightDecay/Mul-op36,kernel_graph0,{8940},8940,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8600,8605,0x12c284e12800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8533,490_Default/optimizer-FP32StateAdamWeightDecay/Square-op7,Default/optimizer-FP32StateAdamWeightDecay/Square-op7,kernel_graph0,{8603},8603,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8603,8947,0x12c20de10600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8534,491_Default/optimizer-FP32StateAdamWeightDecay/Mul-op38,Default/optimizer-FP32StateAdamWeightDecay/Mul-op38,kernel_graph0,{8944},8944,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8606,8911,0x12c284e12800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8535,492_Default/optimizer-FP32StateAdamWeightDecay/Mul-op31,Default/optimizer-FP32StateAdamWeightDecay/Mul-op31,kernel_graph0,{8908},8908,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8608,8613,0x12c28fa12a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8536,493_Default/optimizer-FP32StateAdamWeightDecay/Square-op6,Default/optimizer-FP32StateAdamWeightDecay/Square-op6,kernel_graph0,{8611},8611,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8611,8915,0x12c1d0d35800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8537,494_Default/optimizer-FP32StateAdamWeightDecay/Mul-op33,Default/optimizer-FP32StateAdamWeightDecay/Mul-op33,kernel_graph0,{8912},8912,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8614,8879,0x12c28fa12a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8537,495_Default/optimizer-FP32StateAdamWeightDecay/Mul-op26,Default/optimizer-FP32StateAdamWeightDecay/Mul-op26,kernel_graph0,{8876},8876,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8616,8621,0x12c29a612c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8539,496_Default/optimizer-FP32StateAdamWeightDecay/Square-op5,Default/optimizer-FP32StateAdamWeightDecay/Square-op5,kernel_graph0,{8619},8619,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8619,8883,0x12c1c6135600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8540,497_Default/optimizer-FP32StateAdamWeightDecay/Mul-op28,Default/optimizer-FP32StateAdamWeightDecay/Mul-op28,kernel_graph0,{8880},8880,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8622,8847,0x12c29a612c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8540,498_Default/optimizer-FP32StateAdamWeightDecay/Mul-op21,Default/optimizer-FP32StateAdamWeightDecay/Mul-op21,kernel_graph0,{8844},8844,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8624,8629,0x12c29e612e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8542,499_Default/optimizer-FP32StateAdamWeightDecay/Square-op4,Default/optimizer-FP32StateAdamWeightDecay/Square-op4,kernel_graph0,{8627},8627,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8627,8851,0x12c1f5c14000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8543,500_Default/optimizer-FP32StateAdamWeightDecay/Mul-op23,Default/optimizer-FP32StateAdamWeightDecay/Mul-op23,kernel_graph0,{8848},8848,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8630,8815,0x12c29e612e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8543,501_Default/optimizer-FP32StateAdamWeightDecay/Mul-op16,Default/optimizer-FP32StateAdamWeightDecay/Mul-op16,kernel_graph0,{8812},8812,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8632,8637,0x12c2a2613000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8544,502_Default/optimizer-FP32StateAdamWeightDecay/Square-op3,Default/optimizer-FP32StateAdamWeightDecay/Square-op3,kernel_graph0,{8635},8635,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8635,8819,0x12c1e7936000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8545,503_Default/optimizer-FP32StateAdamWeightDecay/Mul-op18,Default/optimizer-FP32StateAdamWeightDecay/Mul-op18,kernel_graph0,{8816},8816,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8638,8783,0x12c2a2613000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8545,504_Default/optimizer-FP32StateAdamWeightDecay/Mul-op11,Default/optimizer-FP32StateAdamWeightDecay/Mul-op11,kernel_graph0,{8780},8780,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8640,8645,0x12c2a6613200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8547,505_Default/optimizer-FP32StateAdamWeightDecay/Square-op2,Default/optimizer-FP32StateAdamWeightDecay/Square-op2,kernel_graph0,{8643},8643,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8643,8787,0x12c1f1c13e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8548,506_Default/optimizer-FP32StateAdamWeightDecay/Mul-op13,Default/optimizer-FP32StateAdamWeightDecay/Mul-op13,kernel_graph0,{8784},8784,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8646,8751,0x12c2a6613200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8548,507_Default/optimizer-FP32StateAdamWeightDecay/Mul-op6,Default/optimizer-FP32StateAdamWeightDecay/Mul-op6,kernel_graph0,{8748},8748,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8648,8653,0x12c2aa613400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8550,508_Default/optimizer-FP32StateAdamWeightDecay/Square-op1,Default/optimizer-FP32StateAdamWeightDecay/Square-op1,kernel_graph0,{8651},8651,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8651,8755,0x12c208814600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8551,509_Default/optimizer-FP32StateAdamWeightDecay/Mul-op8,Default/optimizer-FP32StateAdamWeightDecay/Mul-op8,kernel_graph0,{8752},8752,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8654,8679,0x12c180008c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8551,510_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Cast-op0,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Cast-op0,kernel_graph0,{8672-8675-8677},8677,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8656,8719,0x12c2aa613400,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8552,511_Default/optimizer-FP32StateAdamWeightDecay/Mul-op1,Default/optimizer-FP32StateAdamWeightDecay/Mul-op1,kernel_graph0,{8716},8716,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8658,8663,0x12c2c9a13600,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8553,512_Default/optimizer-FP32StateAdamWeightDecay/Square-op0,Default/optimizer-FP32StateAdamWeightDecay/Square-op0,kernel_graph0,{8661},8661,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8661,8723,0x12c1a3414200,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8553,513_Default/optimizer-FP32StateAdamWeightDecay/Mul-op3,Default/optimizer-FP32StateAdamWeightDecay/Mul-op3,kernel_graph0,{8720},8720,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8664,9363,0x12c22e221400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8555,514_Default/optimizer-FP32StateAdamWeightDecay/Mul-op101,Default/optimizer-FP32StateAdamWeightDecay/Mul-op101,kernel_graph0,{9360},9360,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8666,8671,0x12c22e225600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8556,515_Default/optimizer-FP32StateAdamWeightDecay/Square-op20,Default/optimizer-FP32StateAdamWeightDecay/Square-op20,kernel_graph0,{8669},8669,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8669,9359,0x12c18000ba00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8556,516_Default/optimizer-FP32StateAdamWeightDecay/Mul-op103,Default/optimizer-FP32StateAdamWeightDecay/Mul-op103,kernel_graph0,{9356},9356,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8672,8674,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8558,518_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/GreaterEqual-op0,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/GreaterEqual-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8675,8680,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8674,522_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Greater-op0,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Greater-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8677,8683,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8675,523_Default/TensorMove-op1,Default/TensorMove-op1,kernel_graph0,{8681},8681,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8681,8686,0x12c180008c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8680,527_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Sub-op1,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Sub-op1,kernel_graph0,{8684},8684,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8684,8689,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8681,528_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Div-op1,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Div-op1,kernel_graph0,{8687},8687,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8687,8692,0x12c180008c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8682,529_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Mul-op1,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Mul-op1,kernel_graph0,{8690},8690,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8690,8695,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8682,530_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Cos-op0,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Cos-op0,kernel_graph0,{8693},8693,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8693,8698,0x12c180008c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8683,531_Default/learning_rate-CosineWithWarmUpLR/Add-op6,Default/learning_rate-CosineWithWarmUpLR/Add-op6,kernel_graph0,{8696},8696,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8696,8701,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8684,532_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Mul-op2,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Mul-op2,kernel_graph0,{8699},8699,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8699,8704,0x12c180008c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8685,533_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Maximum-op0,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Maximum-op0,kernel_graph0,{8702},8702,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8702,8707,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8686,534_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Mul-op3,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Mul-op3,kernel_graph0,{8705},8705,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8705,8710,0x12c180008c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8687,535_Default/learning_rate-CosineWithWarmUpLR/Add-op7,Default/learning_rate-CosineWithWarmUpLR/Add-op7,kernel_graph0,{8708},8708,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8708,9374,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8690,538_Default/optimizer-FP32StateAdamWeightDecay/Reshape-op0,Default/optimizer-FP32StateAdamWeightDecay/Reshape-op0,kernel_graph0,{8732-8764-8796-8828-8860-8892-8924-8956-8988-9020-9052-9084-9116-9148-9180-9212-9244-9276-9308-9340-9372},9372,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8712,8718,0x12c2c9a13600,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8693,540_Default/optimizer-FP32StateAdamWeightDecay/Mul-op0,Default/optimizer-FP32StateAdamWeightDecay/Mul-op0,kernel_graph0,{8716},8716,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8714,8722,0x12c2e8e13800,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8693,541_Default/optimizer-FP32StateAdamWeightDecay/Mul-op2,Default/optimizer-FP32StateAdamWeightDecay/Mul-op2,kernel_graph0,{8720},8720,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8716,8741,0x12c308213a00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8694,542_Default/learning_rate-CosineWithWarmUpLR/Add-op8,Default/learning_rate-CosineWithWarmUpLR/Add-op8,kernel_graph0,{8729-8739},8739,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8720,8739,0x12c2aa613400,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8696,543_Default/learning_rate-CosineWithWarmUpLR/Add-op9,Default/learning_rate-CosineWithWarmUpLR/Add-op9,kernel_graph0,{8724-8737},8737,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8724,8728,0x12c1a3414200,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8696,544_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op0,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op0,kernel_graph0,{8726},8726,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8726,8731,0x12c2c9a13600,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8697,545_Default/learning_rate-CosineWithWarmUpLR/Add-op10,Default/learning_rate-CosineWithWarmUpLR/Add-op10,kernel_graph0,{8729},8729,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8729,8734,0x12c2e8e13800,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8699,546_Default/optimizer-FP32StateAdamWeightDecay/Div-op0,Default/optimizer-FP32StateAdamWeightDecay/Div-op0,kernel_graph0,{8732},8732,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8732,8737,0x12c2c9a13600,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8699,547_Default/optimizer-FP32StateAdamWeightDecay/Mul-op4,Default/optimizer-FP32StateAdamWeightDecay/Mul-op4,kernel_graph0,{8735},8735,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8735,8743,0x12c2e8e13800,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8700,548_Default/optimizer-FP32StateAdamWeightDecay/Sub-op0,Default/optimizer-FP32StateAdamWeightDecay/Sub-op0,kernel_graph0,{8741},8741,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8744,8750,0x12c1a3414200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8703,552_Default/optimizer-FP32StateAdamWeightDecay/Mul-op5,Default/optimizer-FP32StateAdamWeightDecay/Mul-op5,kernel_graph0,{8748},8748,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8746,8754,0x12c1a7414400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8705,553_Default/optimizer-FP32StateAdamWeightDecay/Mul-op7,Default/optimizer-FP32StateAdamWeightDecay/Mul-op7,kernel_graph0,{8752},8752,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8748,8773,0x12c1ab414600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8706,554_Default/learning_rate-CosineWithWarmUpLR/Add-op11,Default/learning_rate-CosineWithWarmUpLR/Add-op11,kernel_graph0,{8761-8771},8771,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8752,8771,0x12c1a3414200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8706,555_Default/learning_rate-CosineWithWarmUpLR/Add-op12,Default/learning_rate-CosineWithWarmUpLR/Add-op12,kernel_graph0,{8756-8769},8769,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8756,8760,0x12c1a7414400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8708,556_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op1,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op1,kernel_graph0,{8758},8758,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8758,8763,0x12c208814600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8709,557_Default/learning_rate-CosineWithWarmUpLR/Add-op13,Default/learning_rate-CosineWithWarmUpLR/Add-op13,kernel_graph0,{8761},8761,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8761,8766,0x12c1a7414400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8710,558_Default/optimizer-FP32StateAdamWeightDecay/Div-op1,Default/optimizer-FP32StateAdamWeightDecay/Div-op1,kernel_graph0,{8764},8764,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8764,8769,0x12c208814600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8710,559_Default/optimizer-FP32StateAdamWeightDecay/Mul-op9,Default/optimizer-FP32StateAdamWeightDecay/Mul-op9,kernel_graph0,{8767},8767,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8767,8775,0x12c1a7414400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8711,560_Default/optimizer-FP32StateAdamWeightDecay/Sub-op1,Default/optimizer-FP32StateAdamWeightDecay/Sub-op1,kernel_graph0,{8773},8773,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8776,8782,0x12c208814600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8712,564_Default/optimizer-FP32StateAdamWeightDecay/Mul-op10,Default/optimizer-FP32StateAdamWeightDecay/Mul-op10,kernel_graph0,{8780},8780,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8778,8786,0x12c1a3414200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8713,565_Default/optimizer-FP32StateAdamWeightDecay/Mul-op12,Default/optimizer-FP32StateAdamWeightDecay/Mul-op12,kernel_graph0,{8784},8784,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8780,8805,0x12c1a7414400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8713,566_Default/learning_rate-CosineWithWarmUpLR/Add-op14,Default/learning_rate-CosineWithWarmUpLR/Add-op14,kernel_graph0,{8793-8803},8803,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8784,8803,0x12c208814600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8714,567_Default/learning_rate-CosineWithWarmUpLR/Add-op15,Default/learning_rate-CosineWithWarmUpLR/Add-op15,kernel_graph0,{8788-8801},8801,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8788,8792,0x12c1a3414200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8715,568_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op2,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op2,kernel_graph0,{8790},8790,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8790,8795,0x12c1f1c13e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8715,569_Default/learning_rate-CosineWithWarmUpLR/Add-op16,Default/learning_rate-CosineWithWarmUpLR/Add-op16,kernel_graph0,{8793},8793,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8793,8798,0x12c1a3414200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8716,570_Default/optimizer-FP32StateAdamWeightDecay/Div-op2,Default/optimizer-FP32StateAdamWeightDecay/Div-op2,kernel_graph0,{8796},8796,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8796,8801,0x12c1f1c13e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8717,571_Default/optimizer-FP32StateAdamWeightDecay/Mul-op14,Default/optimizer-FP32StateAdamWeightDecay/Mul-op14,kernel_graph0,{8799},8799,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8799,8807,0x12c1a3414200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8719,572_Default/optimizer-FP32StateAdamWeightDecay/Sub-op2,Default/optimizer-FP32StateAdamWeightDecay/Sub-op2,kernel_graph0,{8805},8805,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8808,8814,0x12c1f1c13e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8724,576_Default/optimizer-FP32StateAdamWeightDecay/Mul-op15,Default/optimizer-FP32StateAdamWeightDecay/Mul-op15,kernel_graph0,{8812},8812,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8810,8818,0x12c208814600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8725,577_Default/optimizer-FP32StateAdamWeightDecay/Mul-op17,Default/optimizer-FP32StateAdamWeightDecay/Mul-op17,kernel_graph0,{8816},8816,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8812,8837,0x12c1a3414200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8725,578_Default/learning_rate-CosineWithWarmUpLR/Add-op17,Default/learning_rate-CosineWithWarmUpLR/Add-op17,kernel_graph0,{8825-8835},8835,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8816,8835,0x12c1f1c13e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8726,579_Default/learning_rate-CosineWithWarmUpLR/Add-op18,Default/learning_rate-CosineWithWarmUpLR/Add-op18,kernel_graph0,{8820-8833},8833,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8820,8824,0x12c208814600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8727,580_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op3,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op3,kernel_graph0,{8822},8822,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8822,8827,0x12c1e7936000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8727,581_Default/learning_rate-CosineWithWarmUpLR/Add-op19,Default/learning_rate-CosineWithWarmUpLR/Add-op19,kernel_graph0,{8825},8825,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8825,8830,0x12c208814600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8729,582_Default/optimizer-FP32StateAdamWeightDecay/Div-op3,Default/optimizer-FP32StateAdamWeightDecay/Div-op3,kernel_graph0,{8828},8828,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8828,8833,0x12c1e7936000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8730,583_Default/optimizer-FP32StateAdamWeightDecay/Mul-op19,Default/optimizer-FP32StateAdamWeightDecay/Mul-op19,kernel_graph0,{8831},8831,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8831,8839,0x12c208814600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8730,584_Default/optimizer-FP32StateAdamWeightDecay/Sub-op3,Default/optimizer-FP32StateAdamWeightDecay/Sub-op3,kernel_graph0,{8837},8837,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8840,8846,0x12c1f1c13e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8735,588_Default/optimizer-FP32StateAdamWeightDecay/Mul-op20,Default/optimizer-FP32StateAdamWeightDecay/Mul-op20,kernel_graph0,{8844},8844,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8842,8850,0x12c208814600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8736,589_Default/optimizer-FP32StateAdamWeightDecay/Mul-op22,Default/optimizer-FP32StateAdamWeightDecay/Mul-op22,kernel_graph0,{8848},8848,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8844,8869,0x12c1e7936000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8736,590_Default/learning_rate-CosineWithWarmUpLR/Add-op20,Default/learning_rate-CosineWithWarmUpLR/Add-op20,kernel_graph0,{8857-8867},8867,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8848,8867,0x12c1f1c13e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8737,591_Default/learning_rate-CosineWithWarmUpLR/Add-op21,Default/learning_rate-CosineWithWarmUpLR/Add-op21,kernel_graph0,{8852-8865},8865,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8852,8856,0x12c1f5c14000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8738,592_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op4,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op4,kernel_graph0,{8854},8854,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8854,8859,0x12c208814600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8738,593_Default/learning_rate-CosineWithWarmUpLR/Add-op22,Default/learning_rate-CosineWithWarmUpLR/Add-op22,kernel_graph0,{8857},8857,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8857,8862,0x12c1f5c14000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8738,594_Default/optimizer-FP32StateAdamWeightDecay/Div-op4,Default/optimizer-FP32StateAdamWeightDecay/Div-op4,kernel_graph0,{8860},8860,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8860,8865,0x12c208814600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8738,595_Default/optimizer-FP32StateAdamWeightDecay/Mul-op24,Default/optimizer-FP32StateAdamWeightDecay/Mul-op24,kernel_graph0,{8863},8863,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8863,8871,0x12c1f5c14000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8738,596_Default/optimizer-FP32StateAdamWeightDecay/Sub-op4,Default/optimizer-FP32StateAdamWeightDecay/Sub-op4,kernel_graph0,{8869},8869,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8872,8878,0x12c1a3414200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8740,600_Default/optimizer-FP32StateAdamWeightDecay/Mul-op25,Default/optimizer-FP32StateAdamWeightDecay/Mul-op25,kernel_graph0,{8876},8876,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8874,8882,0x12c1ae014400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8741,601_Default/optimizer-FP32StateAdamWeightDecay/Mul-op27,Default/optimizer-FP32StateAdamWeightDecay/Mul-op27,kernel_graph0,{8880},8880,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8876,8901,0x12c1b8c14600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8742,602_Default/learning_rate-CosineWithWarmUpLR/Add-op23,Default/learning_rate-CosineWithWarmUpLR/Add-op23,kernel_graph0,{8889-8899},8899,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8880,8899,0x12c1a3414200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8742,603_Default/learning_rate-CosineWithWarmUpLR/Add-op24,Default/learning_rate-CosineWithWarmUpLR/Add-op24,kernel_graph0,{8884-8897},8897,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8884,8888,0x12c1ae014400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8742,604_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op5,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op5,kernel_graph0,{8886},8886,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8886,8891,0x12c1c3814800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8743,605_Default/learning_rate-CosineWithWarmUpLR/Add-op25,Default/learning_rate-CosineWithWarmUpLR/Add-op25,kernel_graph0,{8889},8889,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8889,8894,0x12c1ae014400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8744,606_Default/optimizer-FP32StateAdamWeightDecay/Div-op5,Default/optimizer-FP32StateAdamWeightDecay/Div-op5,kernel_graph0,{8892},8892,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8892,8897,0x12c1c3814800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8745,607_Default/optimizer-FP32StateAdamWeightDecay/Mul-op29,Default/optimizer-FP32StateAdamWeightDecay/Mul-op29,kernel_graph0,{8895},8895,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8895,8903,0x12c1ae014400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8745,608_Default/optimizer-FP32StateAdamWeightDecay/Sub-op5,Default/optimizer-FP32StateAdamWeightDecay/Sub-op5,kernel_graph0,{8901},8901,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8904,8910,0x12c1a3414200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8748,612_Default/optimizer-FP32StateAdamWeightDecay/Mul-op30,Default/optimizer-FP32StateAdamWeightDecay/Mul-op30,kernel_graph0,{8908},8908,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8906,8914,0x12c1ae014400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8749,613_Default/optimizer-FP32StateAdamWeightDecay/Mul-op32,Default/optimizer-FP32StateAdamWeightDecay/Mul-op32,kernel_graph0,{8912},8912,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8908,8933,0x12c1b8c14600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8749,614_Default/learning_rate-CosineWithWarmUpLR/Add-op26,Default/learning_rate-CosineWithWarmUpLR/Add-op26,kernel_graph0,{8921-8931},8931,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8912,8931,0x12c1a3414200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8752,615_Default/learning_rate-CosineWithWarmUpLR/Add-op27,Default/learning_rate-CosineWithWarmUpLR/Add-op27,kernel_graph0,{8916-8929},8929,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8916,8920,0x12c1ae014400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8753,616_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op6,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op6,kernel_graph0,{8918},8918,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8918,8923,0x12c1c3814800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8753,617_Default/learning_rate-CosineWithWarmUpLR/Add-op28,Default/learning_rate-CosineWithWarmUpLR/Add-op28,kernel_graph0,{8921},8921,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8921,8926,0x12c1ae014400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8756,618_Default/optimizer-FP32StateAdamWeightDecay/Div-op6,Default/optimizer-FP32StateAdamWeightDecay/Div-op6,kernel_graph0,{8924},8924,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8924,8929,0x12c1c3814800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8757,619_Default/optimizer-FP32StateAdamWeightDecay/Mul-op34,Default/optimizer-FP32StateAdamWeightDecay/Mul-op34,kernel_graph0,{8927},8927,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8927,8935,0x12c1ae014400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8757,620_Default/optimizer-FP32StateAdamWeightDecay/Sub-op6,Default/optimizer-FP32StateAdamWeightDecay/Sub-op6,kernel_graph0,{8933},8933,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8936,8942,0x12c1a3414200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8761,624_Default/optimizer-FP32StateAdamWeightDecay/Mul-op35,Default/optimizer-FP32StateAdamWeightDecay/Mul-op35,kernel_graph0,{8940},8940,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8938,8946,0x12c1ae014400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8762,625_Default/optimizer-FP32StateAdamWeightDecay/Mul-op37,Default/optimizer-FP32StateAdamWeightDecay/Mul-op37,kernel_graph0,{8944},8944,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8940,8965,0x12c1b8c14600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8762,626_Default/learning_rate-CosineWithWarmUpLR/Add-op29,Default/learning_rate-CosineWithWarmUpLR/Add-op29,kernel_graph0,{8953-8963},8963,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8944,8963,0x12c1a3414200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8764,627_Default/learning_rate-CosineWithWarmUpLR/Add-op30,Default/learning_rate-CosineWithWarmUpLR/Add-op30,kernel_graph0,{8948-8961},8961,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8948,8952,0x12c1ae014400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8765,628_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op7,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op7,kernel_graph0,{8950},8950,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8950,8955,0x12c208814600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8765,629_Default/learning_rate-CosineWithWarmUpLR/Add-op31,Default/learning_rate-CosineWithWarmUpLR/Add-op31,kernel_graph0,{8953},8953,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8953,8958,0x12c1ae014400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8767,630_Default/optimizer-FP32StateAdamWeightDecay/Div-op7,Default/optimizer-FP32StateAdamWeightDecay/Div-op7,kernel_graph0,{8956},8956,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8956,8961,0x12c208814600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8768,631_Default/optimizer-FP32StateAdamWeightDecay/Mul-op39,Default/optimizer-FP32StateAdamWeightDecay/Mul-op39,kernel_graph0,{8959},8959,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8959,8967,0x12c1ae014400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8768,632_Default/optimizer-FP32StateAdamWeightDecay/Sub-op7,Default/optimizer-FP32StateAdamWeightDecay/Sub-op7,kernel_graph0,{8965},8965,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8968,8974,0x12c1e7936000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8770,636_Default/optimizer-FP32StateAdamWeightDecay/Mul-op40,Default/optimizer-FP32StateAdamWeightDecay/Mul-op40,kernel_graph0,{8972},8972,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8970,8978,0x12c1f1c13e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8771,637_Default/optimizer-FP32StateAdamWeightDecay/Mul-op42,Default/optimizer-FP32StateAdamWeightDecay/Mul-op42,kernel_graph0,{8976},8976,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8972,8997,0x12c1f5c14000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8771,638_Default/learning_rate-CosineWithWarmUpLR/Add-op32,Default/learning_rate-CosineWithWarmUpLR/Add-op32,kernel_graph0,{8985-8995},8995,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8976,8995,0x12c1e7936000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8772,639_Default/learning_rate-CosineWithWarmUpLR/Add-op33,Default/learning_rate-CosineWithWarmUpLR/Add-op33,kernel_graph0,{8980-8993},8993,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8980,8984,0x12c1f1c13e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8772,640_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op8,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op8,kernel_graph0,{8982},8982,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8982,8987,0x12c23a211400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8772,641_Default/learning_rate-CosineWithWarmUpLR/Add-op34,Default/learning_rate-CosineWithWarmUpLR/Add-op34,kernel_graph0,{8985},8985,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8985,8990,0x12c1f1c13e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8773,642_Default/optimizer-FP32StateAdamWeightDecay/Div-op8,Default/optimizer-FP32StateAdamWeightDecay/Div-op8,kernel_graph0,{8988},8988,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8988,8993,0x12c23a211400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8774,643_Default/optimizer-FP32StateAdamWeightDecay/Mul-op44,Default/optimizer-FP32StateAdamWeightDecay/Mul-op44,kernel_graph0,{8991},8991,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 8991,8999,0x12c1f1c13e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8774,644_Default/optimizer-FP32StateAdamWeightDecay/Sub-op8,Default/optimizer-FP32StateAdamWeightDecay/Sub-op8,kernel_graph0,{8997},8997,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9000,9006,0x12c1e7936000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8777,648_Default/optimizer-FP32StateAdamWeightDecay/Mul-op45,Default/optimizer-FP32StateAdamWeightDecay/Mul-op45,kernel_graph0,{9004},9004,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9002,9010,0x12c23a211400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8777,649_Default/optimizer-FP32StateAdamWeightDecay/Mul-op47,Default/optimizer-FP32StateAdamWeightDecay/Mul-op47,kernel_graph0,{9008},9008,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9004,9029,0x12c1f1c13e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8778,650_Default/learning_rate-CosineWithWarmUpLR/Add-op35,Default/learning_rate-CosineWithWarmUpLR/Add-op35,kernel_graph0,{9017-9027},9027,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9008,9027,0x12c1f5c14000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8779,651_Default/learning_rate-CosineWithWarmUpLR/Add-op36,Default/learning_rate-CosineWithWarmUpLR/Add-op36,kernel_graph0,{9012-9025},9025,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9012,9016,0x12c1e7936000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8780,652_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op9,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op9,kernel_graph0,{9014},9014,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9014,9019,0x12c23a211400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8780,653_Default/learning_rate-CosineWithWarmUpLR/Add-op37,Default/learning_rate-CosineWithWarmUpLR/Add-op37,kernel_graph0,{9017},9017,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9017,9022,0x12c1e7936000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8781,654_Default/optimizer-FP32StateAdamWeightDecay/Div-op9,Default/optimizer-FP32StateAdamWeightDecay/Div-op9,kernel_graph0,{9020},9020,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9020,9025,0x12c23a211400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8783,655_Default/optimizer-FP32StateAdamWeightDecay/Mul-op49,Default/optimizer-FP32StateAdamWeightDecay/Mul-op49,kernel_graph0,{9023},9023,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9023,9031,0x12c1e7936000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8784,656_Default/optimizer-FP32StateAdamWeightDecay/Sub-op9,Default/optimizer-FP32StateAdamWeightDecay/Sub-op9,kernel_graph0,{9029},9029,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9032,9038,0x12c1e7936000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8789,660_Default/optimizer-FP32StateAdamWeightDecay/Mul-op50,Default/optimizer-FP32StateAdamWeightDecay/Mul-op50,kernel_graph0,{9036},9036,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9034,9042,0x12c23a211400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8789,661_Default/optimizer-FP32StateAdamWeightDecay/Mul-op52,Default/optimizer-FP32StateAdamWeightDecay/Mul-op52,kernel_graph0,{9040},9040,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9036,9061,0x12c1f1c13e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8790,662_Default/learning_rate-CosineWithWarmUpLR/Add-op38,Default/learning_rate-CosineWithWarmUpLR/Add-op38,kernel_graph0,{9049-9059},9059,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9040,9059,0x12c1f5c14000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8791,663_Default/learning_rate-CosineWithWarmUpLR/Add-op39,Default/learning_rate-CosineWithWarmUpLR/Add-op39,kernel_graph0,{9044-9057},9057,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9044,9048,0x12c23a211400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8793,664_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op10,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op10,kernel_graph0,{9046},9046,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9046,9051,0x12c1e3935e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8793,665_Default/learning_rate-CosineWithWarmUpLR/Add-op40,Default/learning_rate-CosineWithWarmUpLR/Add-op40,kernel_graph0,{9049},9049,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9049,9054,0x12c23a211400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8794,666_Default/optimizer-FP32StateAdamWeightDecay/Div-op10,Default/optimizer-FP32StateAdamWeightDecay/Div-op10,kernel_graph0,{9052},9052,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9052,9057,0x12c1e3935e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8795,667_Default/optimizer-FP32StateAdamWeightDecay/Mul-op54,Default/optimizer-FP32StateAdamWeightDecay/Mul-op54,kernel_graph0,{9055},9055,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9055,9063,0x12c23a211400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8796,668_Default/optimizer-FP32StateAdamWeightDecay/Sub-op10,Default/optimizer-FP32StateAdamWeightDecay/Sub-op10,kernel_graph0,{9061},9061,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9064,9070,0x12c23a211400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8800,672_Default/optimizer-FP32StateAdamWeightDecay/Mul-op55,Default/optimizer-FP32StateAdamWeightDecay/Mul-op55,kernel_graph0,{9068},9068,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9066,9074,0x12c1f1c13e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8801,673_Default/optimizer-FP32StateAdamWeightDecay/Mul-op57,Default/optimizer-FP32StateAdamWeightDecay/Mul-op57,kernel_graph0,{9072},9072,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9068,9093,0x12c1f5c14000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8802,674_Default/learning_rate-CosineWithWarmUpLR/Add-op41,Default/learning_rate-CosineWithWarmUpLR/Add-op41,kernel_graph0,{9081-9091},9091,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9072,9091,0x12c23a211400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8802,675_Default/learning_rate-CosineWithWarmUpLR/Add-op42,Default/learning_rate-CosineWithWarmUpLR/Add-op42,kernel_graph0,{9076-9089},9089,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9076,9080,0x12c1f1c13e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8802,676_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op11,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op11,kernel_graph0,{9078},9078,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9078,9083,0x12c204814400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8803,677_Default/learning_rate-CosineWithWarmUpLR/Add-op43,Default/learning_rate-CosineWithWarmUpLR/Add-op43,kernel_graph0,{9081},9081,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9081,9086,0x12c1f1c13e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8803,678_Default/optimizer-FP32StateAdamWeightDecay/Div-op11,Default/optimizer-FP32StateAdamWeightDecay/Div-op11,kernel_graph0,{9084},9084,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9084,9089,0x12c204814400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8804,679_Default/optimizer-FP32StateAdamWeightDecay/Mul-op59,Default/optimizer-FP32StateAdamWeightDecay/Mul-op59,kernel_graph0,{9087},9087,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9087,9095,0x12c1f1c13e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8804,680_Default/optimizer-FP32StateAdamWeightDecay/Sub-op11,Default/optimizer-FP32StateAdamWeightDecay/Sub-op11,kernel_graph0,{9093},9093,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9096,9102,0x12c204814400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8806,684_Default/optimizer-FP32StateAdamWeightDecay/Mul-op60,Default/optimizer-FP32StateAdamWeightDecay/Mul-op60,kernel_graph0,{9100},9100,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9098,9106,0x12c19f414000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8806,685_Default/optimizer-FP32StateAdamWeightDecay/Mul-op62,Default/optimizer-FP32StateAdamWeightDecay/Mul-op62,kernel_graph0,{9104},9104,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9100,9125,0x12c1aa014200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8808,686_Default/learning_rate-CosineWithWarmUpLR/Add-op44,Default/learning_rate-CosineWithWarmUpLR/Add-op44,kernel_graph0,{9113-9123},9123,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9104,9123,0x12c204814400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8808,687_Default/learning_rate-CosineWithWarmUpLR/Add-op45,Default/learning_rate-CosineWithWarmUpLR/Add-op45,kernel_graph0,{9108-9121},9121,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9108,9112,0x12c19f414000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8809,688_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op12,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op12,kernel_graph0,{9110},9110,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9110,9115,0x12c223621200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8809,689_Default/learning_rate-CosineWithWarmUpLR/Add-op46,Default/learning_rate-CosineWithWarmUpLR/Add-op46,kernel_graph0,{9113},9113,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9113,9118,0x12c19f414000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8810,690_Default/optimizer-FP32StateAdamWeightDecay/Div-op12,Default/optimizer-FP32StateAdamWeightDecay/Div-op12,kernel_graph0,{9116},9116,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9116,9121,0x12c223621200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8811,691_Default/optimizer-FP32StateAdamWeightDecay/Mul-op64,Default/optimizer-FP32StateAdamWeightDecay/Mul-op64,kernel_graph0,{9119},9119,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9119,9127,0x12c19f414000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8811,692_Default/optimizer-FP32StateAdamWeightDecay/Sub-op12,Default/optimizer-FP32StateAdamWeightDecay/Sub-op12,kernel_graph0,{9125},9125,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9128,9134,0x12c223621200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8816,696_Default/optimizer-FP32StateAdamWeightDecay/Mul-op65,Default/optimizer-FP32StateAdamWeightDecay/Mul-op65,kernel_graph0,{9132},9132,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9130,9138,0x12c204814400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8817,697_Default/optimizer-FP32StateAdamWeightDecay/Mul-op67,Default/optimizer-FP32StateAdamWeightDecay/Mul-op67,kernel_graph0,{9136},9136,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9132,9157,0x12c19f414000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8817,698_Default/learning_rate-CosineWithWarmUpLR/Add-op47,Default/learning_rate-CosineWithWarmUpLR/Add-op47,kernel_graph0,{9145-9155},9155,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9136,9155,0x12c223621200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8820,699_Default/learning_rate-CosineWithWarmUpLR/Add-op48,Default/learning_rate-CosineWithWarmUpLR/Add-op48,kernel_graph0,{9140-9153},9153,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9140,9144,0x12c22e225600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8821,700_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op13,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op13,kernel_graph0,{9142},9142,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9142,9147,0x12c204814400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8821,701_Default/learning_rate-CosineWithWarmUpLR/Add-op49,Default/learning_rate-CosineWithWarmUpLR/Add-op49,kernel_graph0,{9145},9145,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9145,9150,0x12c22e225600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8822,702_Default/optimizer-FP32StateAdamWeightDecay/Div-op13,Default/optimizer-FP32StateAdamWeightDecay/Div-op13,kernel_graph0,{9148},9148,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9148,9153,0x12c204814400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8823,703_Default/optimizer-FP32StateAdamWeightDecay/Mul-op69,Default/optimizer-FP32StateAdamWeightDecay/Mul-op69,kernel_graph0,{9151},9151,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9151,9159,0x12c22e225600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8823,704_Default/optimizer-FP32StateAdamWeightDecay/Sub-op13,Default/optimizer-FP32StateAdamWeightDecay/Sub-op13,kernel_graph0,{9157},9157,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9160,9166,0x12c223621200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8827,708_Default/optimizer-FP32StateAdamWeightDecay/Mul-op70,Default/optimizer-FP32StateAdamWeightDecay/Mul-op70,kernel_graph0,{9164},9164,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9162,9170,0x12c22e225600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8828,709_Default/optimizer-FP32StateAdamWeightDecay/Mul-op72,Default/optimizer-FP32StateAdamWeightDecay/Mul-op72,kernel_graph0,{9168},9168,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9164,9189,0x12c204814400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8829,710_Default/learning_rate-CosineWithWarmUpLR/Add-op50,Default/learning_rate-CosineWithWarmUpLR/Add-op50,kernel_graph0,{9177-9187},9187,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9168,9187,0x12c223621200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8830,711_Default/learning_rate-CosineWithWarmUpLR/Add-op51,Default/learning_rate-CosineWithWarmUpLR/Add-op51,kernel_graph0,{9172-9185},9185,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9172,9176,0x12c1f1c13e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8831,712_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op14,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op14,kernel_graph0,{9174},9174,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9174,9179,0x12c22e225600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8832,713_Default/learning_rate-CosineWithWarmUpLR/Add-op52,Default/learning_rate-CosineWithWarmUpLR/Add-op52,kernel_graph0,{9177},9177,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9177,9182,0x12c1f1c13e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8833,714_Default/optimizer-FP32StateAdamWeightDecay/Div-op14,Default/optimizer-FP32StateAdamWeightDecay/Div-op14,kernel_graph0,{9180},9180,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9180,9185,0x12c22e225600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8833,715_Default/optimizer-FP32StateAdamWeightDecay/Mul-op74,Default/optimizer-FP32StateAdamWeightDecay/Mul-op74,kernel_graph0,{9183},9183,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9183,9191,0x12c1f1c13e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8834,716_Default/optimizer-FP32StateAdamWeightDecay/Sub-op14,Default/optimizer-FP32StateAdamWeightDecay/Sub-op14,kernel_graph0,{9189},9189,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9192,9198,0x12c1f1c13e00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8836,720_Default/optimizer-FP32StateAdamWeightDecay/Mul-op75,Default/optimizer-FP32StateAdamWeightDecay/Mul-op75,kernel_graph0,{9196},9196,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9194,9202,0x12c19f414000,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8836,721_Default/optimizer-FP32StateAdamWeightDecay/Mul-op77,Default/optimizer-FP32StateAdamWeightDecay/Mul-op77,kernel_graph0,{9200},9200,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9196,9221,0x12c1be814200,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8836,722_Default/learning_rate-CosineWithWarmUpLR/Add-op53,Default/learning_rate-CosineWithWarmUpLR/Add-op53,kernel_graph0,{9209-9219},9219,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9200,9219,0x12c1f1c13e00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8837,723_Default/learning_rate-CosineWithWarmUpLR/Add-op54,Default/learning_rate-CosineWithWarmUpLR/Add-op54,kernel_graph0,{9204-9217},9217,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9204,9208,0x12c180013e00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8838,724_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op15,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op15,kernel_graph0,{9206},9206,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9206,9211,0x12c19f414000,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8838,725_Default/learning_rate-CosineWithWarmUpLR/Add-op55,Default/learning_rate-CosineWithWarmUpLR/Add-op55,kernel_graph0,{9209},9209,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9209,9214,0x12c180013e00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8838,726_Default/optimizer-FP32StateAdamWeightDecay/Div-op15,Default/optimizer-FP32StateAdamWeightDecay/Div-op15,kernel_graph0,{9212},9212,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9212,9217,0x12c19f414000,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8840,727_Default/optimizer-FP32StateAdamWeightDecay/Mul-op79,Default/optimizer-FP32StateAdamWeightDecay/Mul-op79,kernel_graph0,{9215},9215,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9215,9223,0x12c180013e00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8840,728_Default/optimizer-FP32StateAdamWeightDecay/Sub-op15,Default/optimizer-FP32StateAdamWeightDecay/Sub-op15,kernel_graph0,{9221},9221,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9224,9230,0x12c223621200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8843,732_Default/optimizer-FP32StateAdamWeightDecay/Mul-op80,Default/optimizer-FP32StateAdamWeightDecay/Mul-op80,kernel_graph0,{9228},9228,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9226,9234,0x12c223625400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8843,733_Default/optimizer-FP32StateAdamWeightDecay/Mul-op82,Default/optimizer-FP32StateAdamWeightDecay/Mul-op82,kernel_graph0,{9232},9232,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9228,9253,0x12c223629600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8844,734_Default/learning_rate-CosineWithWarmUpLR/Add-op56,Default/learning_rate-CosineWithWarmUpLR/Add-op56,kernel_graph0,{9241-9251},9251,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9232,9251,0x12c22361d000,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8845,735_Default/learning_rate-CosineWithWarmUpLR/Add-op57,Default/learning_rate-CosineWithWarmUpLR/Add-op57,kernel_graph0,{9236-9249},9249,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9236,9240,0x12c223621200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8845,736_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op16,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op16,kernel_graph0,{9238},9238,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9238,9243,0x12c223625400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8848,737_Default/learning_rate-CosineWithWarmUpLR/Add-op58,Default/learning_rate-CosineWithWarmUpLR/Add-op58,kernel_graph0,{9241},9241,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9241,9246,0x12c223621200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8849,738_Default/optimizer-FP32StateAdamWeightDecay/Div-op16,Default/optimizer-FP32StateAdamWeightDecay/Div-op16,kernel_graph0,{9244},9244,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9244,9249,0x12c223625400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8849,739_Default/optimizer-FP32StateAdamWeightDecay/Mul-op84,Default/optimizer-FP32StateAdamWeightDecay/Mul-op84,kernel_graph0,{9247},9247,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9247,9255,0x12c223621200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8852,740_Default/optimizer-FP32StateAdamWeightDecay/Sub-op16,Default/optimizer-FP32StateAdamWeightDecay/Sub-op16,kernel_graph0,{9253},9253,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9256,9262,0x12c22361d000,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8855,744_Default/optimizer-FP32StateAdamWeightDecay/Mul-op85,Default/optimizer-FP32StateAdamWeightDecay/Mul-op85,kernel_graph0,{9260},9260,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9258,9266,0x12c223621200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8856,745_Default/optimizer-FP32StateAdamWeightDecay/Mul-op87,Default/optimizer-FP32StateAdamWeightDecay/Mul-op87,kernel_graph0,{9264},9264,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9260,9285,0x12c223625400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8857,746_Default/learning_rate-CosineWithWarmUpLR/Add-op59,Default/learning_rate-CosineWithWarmUpLR/Add-op59,kernel_graph0,{9273-9283},9283,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9264,9283,0x12c223618e00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8858,747_Default/learning_rate-CosineWithWarmUpLR/Add-op60,Default/learning_rate-CosineWithWarmUpLR/Add-op60,kernel_graph0,{9268-9281},9281,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9268,9272,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8859,748_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op17,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op17,kernel_graph0,{9270},9270,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9270,9275,0x12c22361d000,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8860,749_Default/learning_rate-CosineWithWarmUpLR/Add-op61,Default/learning_rate-CosineWithWarmUpLR/Add-op61,kernel_graph0,{9273},9273,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9273,9278,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8861,750_Default/optimizer-FP32StateAdamWeightDecay/Div-op17,Default/optimizer-FP32StateAdamWeightDecay/Div-op17,kernel_graph0,{9276},9276,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9276,9281,0x12c22361d000,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8861,751_Default/optimizer-FP32StateAdamWeightDecay/Mul-op89,Default/optimizer-FP32StateAdamWeightDecay/Mul-op89,kernel_graph0,{9279},9279,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9279,9287,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8863,752_Default/optimizer-FP32StateAdamWeightDecay/Sub-op17,Default/optimizer-FP32StateAdamWeightDecay/Sub-op17,kernel_graph0,{9285},9285,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9288,9294,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8866,756_Default/optimizer-FP32StateAdamWeightDecay/Mul-op90,Default/optimizer-FP32StateAdamWeightDecay/Mul-op90,kernel_graph0,{9292},9292,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9290,9298,0x12c223618e00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8866,757_Default/optimizer-FP32StateAdamWeightDecay/Mul-op92,Default/optimizer-FP32StateAdamWeightDecay/Mul-op92,kernel_graph0,{9296},9296,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9292,9317,0x12c22361d000,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8866,758_Default/learning_rate-CosineWithWarmUpLR/Add-op62,Default/learning_rate-CosineWithWarmUpLR/Add-op62,kernel_graph0,{9305-9315},9315,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9296,9315,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8867,759_Default/learning_rate-CosineWithWarmUpLR/Add-op63,Default/learning_rate-CosineWithWarmUpLR/Add-op63,kernel_graph0,{9300-9313},9313,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9300,9304,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8867,760_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op18,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op18,kernel_graph0,{9302},9302,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9302,9307,0x12c223621200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8868,761_Default/learning_rate-CosineWithWarmUpLR/Add-op64,Default/learning_rate-CosineWithWarmUpLR/Add-op64,kernel_graph0,{9305},9305,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9305,9310,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8868,762_Default/optimizer-FP32StateAdamWeightDecay/Div-op18,Default/optimizer-FP32StateAdamWeightDecay/Div-op18,kernel_graph0,{9308},9308,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9308,9313,0x12c223621200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8868,763_Default/optimizer-FP32StateAdamWeightDecay/Mul-op94,Default/optimizer-FP32StateAdamWeightDecay/Mul-op94,kernel_graph0,{9311},9311,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9311,9319,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8869,764_Default/optimizer-FP32StateAdamWeightDecay/Sub-op18,Default/optimizer-FP32StateAdamWeightDecay/Sub-op18,kernel_graph0,{9317},9317,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9320,9326,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8870,768_Default/optimizer-FP32StateAdamWeightDecay/Mul-op97,Default/optimizer-FP32StateAdamWeightDecay/Mul-op97,kernel_graph0,{9324},9324,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9322,9330,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8872,769_Default/optimizer-FP32StateAdamWeightDecay/Mul-op95,Default/optimizer-FP32StateAdamWeightDecay/Mul-op95,kernel_graph0,{9328},9328,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9324,9347,0x12c1f1c13e00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8873,770_Default/learning_rate-CosineWithWarmUpLR/Add-op66,Default/learning_rate-CosineWithWarmUpLR/Add-op66,kernel_graph0,{9332-9345},9345,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9328,9349,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8873,771_Default/learning_rate-CosineWithWarmUpLR/Add-op65,Default/learning_rate-CosineWithWarmUpLR/Add-op65,kernel_graph0,{9337-9347},9347,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9332,9336,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8874,772_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op19,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op19,kernel_graph0,{9334},9334,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9334,9339,0x12c1f1c18000,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8875,773_Default/learning_rate-CosineWithWarmUpLR/Add-op67,Default/learning_rate-CosineWithWarmUpLR/Add-op67,kernel_graph0,{9337},9337,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9337,9342,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8875,774_Default/optimizer-FP32StateAdamWeightDecay/Div-op19,Default/optimizer-FP32StateAdamWeightDecay/Div-op19,kernel_graph0,{9340},9340,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9340,9345,0x12c1f1c18000,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8876,775_Default/optimizer-FP32StateAdamWeightDecay/Mul-op99,Default/optimizer-FP32StateAdamWeightDecay/Mul-op99,kernel_graph0,{9343},9343,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9343,9351,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8877,776_Default/optimizer-FP32StateAdamWeightDecay/Sub-op19,Default/optimizer-FP32StateAdamWeightDecay/Sub-op19,kernel_graph0,{9349},9349,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9352,9358,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8881,780_Default/optimizer-FP32StateAdamWeightDecay/Mul-op102,Default/optimizer-FP32StateAdamWeightDecay/Mul-op102,kernel_graph0,{9356},9356,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9354,9362,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8884,781_Default/optimizer-FP32StateAdamWeightDecay/Mul-op100,Default/optimizer-FP32StateAdamWeightDecay/Mul-op100,kernel_graph0,{9360},9360,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9356,9384,0x12c18000fc00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8885,782_Default/learning_rate-CosineWithWarmUpLR/Add-op69,Default/learning_rate-CosineWithWarmUpLR/Add-op69,kernel_graph0,{9364-9382},9382,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9360,9382,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8885,783_Default/learning_rate-CosineWithWarmUpLR/Add-op68,Default/learning_rate-CosineWithWarmUpLR/Add-op68,kernel_graph0,{9369-9380},9380,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9364,9368,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8886,784_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op20,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op20,kernel_graph0,{9366},9366,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9366,9371,0x12c18000ba00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8887,785_Default/learning_rate-CosineWithWarmUpLR/Add-op70,Default/learning_rate-CosineWithWarmUpLR/Add-op70,kernel_graph0,{9369},9369,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9369,9375,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8887,786_Default/optimizer-FP32StateAdamWeightDecay/Div-op20,Default/optimizer-FP32StateAdamWeightDecay/Div-op20,kernel_graph0,{9372},9372,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9372,9378,0x12c18000ba00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8889,787_Default/optimizer-FP32StateAdamWeightDecay/Mul-op104,Default/optimizer-FP32StateAdamWeightDecay/Mul-op104,kernel_graph0,{9376},9376,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9376,9380,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8890,788_Default/optimizer-FP32StateAdamWeightDecay/Sub-op20,Default/optimizer-FP32StateAdamWeightDecay/Sub-op20,kernel_graph0,{9378},9378,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9385,11684,0x12c180008c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,8893,792_Default/TensorMove-op2,Default/TensorMove-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9389,11682,0x12c75d35ee00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/output_actor.cc,689,Other,9387,kernel_graph_0_OutputActor,UpdateOutputDeviceAddress,,{},,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 9395,9799,0x12c75d35f000,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Kernel,9394,0_Default/network-LlamaForCausalLM/StridedSlice-op0,inputs0,0_Default/network-LlamaForCausalLM/StridedSlice-op0,{9396-9797},9797,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,1,0, 9396,10378,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9394,0_Default/network-LlamaForCausalLM/StridedSlice-op0,inputs0,0_Default/network-LlamaForCausalLM/StridedSlice-op0,{9398-9400-10375},10375,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9398,9404,0x12c18000ba00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9396,1_Default/network-LlamaForCausalLM/model-LlamaModel/tok_embeddings-LlamaEmbedding/Gather-op0,Default/network-LlamaForCausalLM/model-LlamaModel/tok_embeddings-LlamaEmbedding/Gather-op0,kernel_graph0,{9402},9402,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9400,9410,0x12c18800bc00,0,DefaultEnhancedAscendMemoryPool,8704,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9397,2_Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/NotEqual-op0,Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/NotEqual-op0,kernel_graph0,{9405-9408},9408,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9402,10356,0x12c18800de00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9397,3_Default/network-LlamaForCausalLM/model-LlamaModel/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/Cast-op0,kernel_graph0,{9414-9543-10354},10354,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9405,9833,0x12c18000ba00,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9397,4_Default/network-LlamaForCausalLM/Cast-op1,Default/network-LlamaForCausalLM/Cast-op1,kernel_graph0,{9406-9425-9786-9830},9830,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9408,9439,0x12c180013c00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9398,6_Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Cast-op0,kernel_graph0,{9433-9437},9437,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9411,10282,0x12c180017e00,0,DefaultEnhancedAscendMemoryPool,524800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9398,7_Default/network-LlamaForCausalLM/model-LlamaModel/freqs_mgr-FreqsMgr/StridedSlice-op0,Default/network-LlamaForCausalLM/model-LlamaModel/freqs_mgr-FreqsMgr/StridedSlice-op0,kernel_graph0,{9412-9476-9499-9629-9644-10017-10038-10259-10280},10280,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9414,9420,0x12c18c00e000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9399,9_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op0,kernel_graph0,{9418},9418,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9416,9424,0x12c19400e200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9399,10_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{9422},9422,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9417,10363,0x12c180098000,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9399,10_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{10359},10359,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9418,9421,0x12c1800a0200,0,DefaultEnhancedAscendMemoryPool,16777728,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9399,10_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9422,10334,0x12c1800a0200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9400,11_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op1,kernel_graph0,{9447-9452-9467-9490-10304-10318-10332},10332,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9425,9433,0x12c180008800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9401,12_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/ReduceSum-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/ReduceSum-op0,kernel_graph0,{9431},9431,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9427,9429,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,9427,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9428,9430,0x12c180009200,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,9428,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9431,9844,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9401,13_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Add-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Add-op0,kernel_graph0,{9774-9841},9841,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9435,9440,0x12c1840a0400,0,DefaultEnhancedAscendMemoryPool,2097664,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9403,15_Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/ExpandDims-op0,Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/ExpandDims-op0,kernel_graph0,{9437},9437,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9437,9443,0x12c1842a0600,0,DefaultEnhancedAscendMemoryPool,16777728,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9403,16_Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Mul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Mul-op0,kernel_graph0,{9441},9441,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9441,9447,0x12c1852a0800,0,DefaultEnhancedAscendMemoryPool,16777728,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9405,17_Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Sub-op0,Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Sub-op0,kernel_graph0,{9443-9445},9445,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9445,10240,0x12c1840a0400,0,DefaultEnhancedAscendMemoryPool,8389120,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9406,19_Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Cast-op1,kernel_graph0,{9528-9675-9993-10234},10234,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9449,10328,0x12c1848a0600,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9406,21_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/Cast-op0,kernel_graph0,{9452-10325},10325,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9451,9459,0x12c18c00e000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9407,22_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,kernel_graph0,{9454-9457},9457,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9452,9454,0x12c1868a0800,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9407,22_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9456,9463,0x12c19000e200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9409,24_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,kernel_graph0,{9461},9461,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9457,9460,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9409,24_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9461,9480,0x12c19400e400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9409,25_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op2,kernel_graph0,{9476-9478},9478,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9464,10343,0x12c18c00e000,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9411,26_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/Cast-op0,kernel_graph0,{9467-10340},10340,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9466,9474,0x12c18e00e200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9412,27_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,kernel_graph0,{9469-9472},9472,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9467,9469,0x12c1868a0800,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9412,27_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9471,9483,0x12c19c00e600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9412,29_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,kernel_graph0,{9481},9481,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9472,9475,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9412,29_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9476,9519,0x12c1a000e800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9412,30_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op2,kernel_graph0,{9517},9517,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9478,9516,0x12c1a800ea00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9412,31_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op1,kernel_graph0,{9514},9514,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9481,9503,0x12c18e00e200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9413,32_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op0,kernel_graph0,{9499-9501},9501,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9484,10285,0x12c1868a0800,0,DefaultEnhancedAscendMemoryPool,524800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9414,33_Default/network-LlamaForCausalLM/model-LlamaModel/freqs_mgr-FreqsMgr/StridedSlice-op1,Default/network-LlamaForCausalLM/model-LlamaModel/freqs_mgr-FreqsMgr/StridedSlice-op1,kernel_graph0,{9485-9504-9514-9634-9649-10019-10040-10261-10283},10283,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9487,10314,0x12c19600e400,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9415,35_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/Cast-op0,kernel_graph0,{9490-10311},10311,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9489,9497,0x12c19800e600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9418,36_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,kernel_graph0,{9492-9495},9495,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9490,9492,0x12c186920a00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9418,36_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9494,10238,0x12c1b000ec00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9419,38_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,kernel_graph0,{9528-10234},10234,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9495,9498,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9419,38_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9499,9509,0x12c19800e600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9421,39_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op0,kernel_graph0,{9507},9507,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9501,9506,0x12c1b400ee00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9422,40_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op0,kernel_graph0,{9504},9504,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9504,9510,0x12c18e00e200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9423,41_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op1,kernel_graph0,{9507},9507,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9507,9513,0x12c1b400ee00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9425,42_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op0,kernel_graph0,{9511},9511,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9511,10236,0x12c18e00e200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9425,43_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op1,kernel_graph0,{9528-10234},10234,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9514,9520,0x12c19800e600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9426,44_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op3,kernel_graph0,{9517},9517,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9517,9523,0x12c1a800ea00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9426,45_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op1,kernel_graph0,{9521},9521,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9521,10237,0x12c19800e600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9426,46_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op3,kernel_graph0,{9528-10234},10234,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9524,10241,0x12c186920a00,0,DefaultEnhancedAscendMemoryPool,8389120,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9427,47_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{10234},10234,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9525,10242,0x12c187120c00,0,DefaultEnhancedAscendMemoryPool,8389120,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9427,47_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{10234},10234,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9526,10243,0x12c180008800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9427,47_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{10234},10234,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9527,10244,0x12c19c00e800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9427,47_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{9532-10234},10234,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9528,9530,0x12c1a000ea00,0,DefaultEnhancedAscendMemoryPool,84936192,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9427,47_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9531,10214,0x12c1a000ea00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9427,48_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,kernel_graph0,{9534-9539-10212},10212,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9532,9534,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9427,48_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9536,10222,0x12c19200e400,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9428,50_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/Cast-op0,kernel_graph0,{9539-10220},10220,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9538,9545,0x12c1a400ec00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9431,51_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,kernel_graph0,{9541-9543},9543,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9539,9541,0x12c19400e600,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9431,51_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9543,10194,0x12c1a800ee00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9432,53_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/Add-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/Add-op0,kernel_graph0,{9546-9584-10192},10192,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9546,9552,0x12c1b400ee00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9433,54_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op0,kernel_graph0,{9550},9550,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9548,9556,0x12c1bc00f000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9433,55_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{9554},9554,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9549,10201,0x12c187920e00,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9433,55_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{10197},10197,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9550,9553,0x12c19400e600,0,DefaultEnhancedAscendMemoryPool,16777728,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9433,55_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9554,10173,0x12c1a400ec00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9434,56_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op1,kernel_graph0,{9556-9561-9568-10157-10171},10171,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9558,10167,0x12c1b400ee00,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9436,58_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/Cast-op0,kernel_graph0,{9561-10164},10164,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9560,10149,0x12c1b960f000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9437,59_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,kernel_graph0,{9563-9574-10147},10147,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9561,9563,0x12c19400e600,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9437,59_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9565,10182,0x12c1c420f200,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9438,61_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/Cast-op0,kernel_graph0,{9568-10179},10179,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9567,10155,0x12c1c980f400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9441,62_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,kernel_graph0,{9571-10152},10152,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9568,9570,0x12c19400e600,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9441,62_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9571,10145,0x12c1d440f600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9442,63_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/SiLU-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/SiLU-op0,kernel_graph0,{9572-9574-10143},10143,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9574,10132,0x12c1df00f800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9443,65_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/Mul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/Mul-op0,kernel_graph0,{9575-9580-10130},10130,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9577,10140,0x12c1e9c0fa00,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9445,67_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/Cast-op0,kernel_graph0,{9580-10138},10138,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9579,9586,0x12c1ef20fc00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9446,68_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,kernel_graph0,{9582-9584},9584,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9580,9582,0x12c19400e600,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9446,68_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9584,10112,0x12c1f320fe00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9447,70_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/Add-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/Add-op1,kernel_graph0,{9587-9690-10110},10110,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9587,9593,0x12c1f7210000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9448,71_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op0,kernel_graph0,{9591},9591,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9589,9597,0x12c1ff210200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9449,72_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{9595},9595,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9590,10119,0x12c187929000,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9449,72_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{10115},10115,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9591,9594,0x12c19400e600,0,DefaultEnhancedAscendMemoryPool,16777728,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9449,72_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9595,10090,0x12c1ef20fc00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9450,73_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op1,kernel_graph0,{9597-9602-9617-9662-10060-10074-10088},10088,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9599,10084,0x12c1ac00f000,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9452,75_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/Cast-op0,kernel_graph0,{9602-10081},10081,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9601,9609,0x12c1f7210000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9453,76_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,kernel_graph0,{9604-9607},9607,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9602,9604,0x12c1ae00f200,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9453,76_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9606,9613,0x12c1fb210200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9454,78_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,kernel_graph0,{9611},9611,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9607,9610,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9454,78_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9611,9633,0x12c1ff210400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9455,79_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op2,kernel_graph0,{9629-9631},9631,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9614,10099,0x12c1f7210000,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9457,80_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/Cast-op0,kernel_graph0,{9617-10096},10096,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9616,9624,0x12c1f9210200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9458,81_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,kernel_graph0,{9619-9622},9622,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9617,9619,0x12c1ae00f200,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9458,81_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9621,9628,0x12c207210600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9460,83_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,kernel_graph0,{9626},9626,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9622,9625,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9460,83_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9626,9648,0x12c20b210800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9461,84_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op0,kernel_graph0,{9644-9646},9646,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9629,9639,0x12c213210a00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9462,85_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op2,kernel_graph0,{9637},9637,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9631,9636,0x12c21b210c00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9462,86_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op1,kernel_graph0,{9634},9634,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9634,9640,0x12c1f9210200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9464,87_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op3,kernel_graph0,{9637},9637,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9637,9643,0x12c201210400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9465,88_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op1,kernel_graph0,{9641},9641,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9641,9996,0x12c1f9210200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9465,89_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op3,kernel_graph0,{9675-9993},9993,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9644,9654,0x12c1fd210400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9467,90_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op0,kernel_graph0,{9652},9652,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9646,9651,0x12c213210a00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9468,91_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op0,kernel_graph0,{9649},9649,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9649,9655,0x12c205210600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9468,92_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op1,kernel_graph0,{9652},9652,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9652,9658,0x12c20d210800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9469,93_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op0,kernel_graph0,{9656},9656,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9656,9995,0x12c1fd210400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9470,94_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op1,kernel_graph0,{9675-9993},9993,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9659,10070,0x12c201210600,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9472,95_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/Cast-op0,kernel_graph0,{9662-10067},10067,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9661,9669,0x12c203210800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9473,96_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,kernel_graph0,{9664-9667},9667,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9662,9664,0x12c1ae00f200,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9473,96_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9666,9997,0x12c207210a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9476,98_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,kernel_graph0,{9675-9993},9993,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9667,9670,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9476,98_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9671,9999,0x12c1ae00f200,0,DefaultEnhancedAscendMemoryPool,8389120,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9477,99_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{9993},9993,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9672,10000,0x12c1ae80f400,0,DefaultEnhancedAscendMemoryPool,8389120,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9477,99_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{9993},9993,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9673,10001,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9477,99_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{9993},9993,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9674,10002,0x12c203210800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9477,99_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{9679-9993},9993,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9675,9677,0x12c20b210c00,0,DefaultEnhancedAscendMemoryPool,84936192,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9477,99_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9678,9973,0x12c20b210c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9477,100_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,kernel_graph0,{9681-9686-9971},9971,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9679,9681,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9477,100_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9683,9981,0x12c20f210e00,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9478,102_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/Cast-op0,kernel_graph0,{9686-9979},9979,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9685,9692,0x12c211211000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9479,103_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,kernel_graph0,{9688-9690},9690,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9686,9688,0x12c19400e600,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9479,103_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9690,9948,0x12c215211200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9481,105_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/Add-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/Add-op0,kernel_graph0,{9693-9731-9946},9946,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9693,9699,0x12c219211400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9482,106_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op0,kernel_graph0,{9697},9697,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9695,9703,0x12c221211600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9482,107_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{9701},9701,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9696,9955,0x12c187931200,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9482,107_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{9951},9951,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9697,9700,0x12c19400e600,0,DefaultEnhancedAscendMemoryPool,16777728,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9482,107_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9701,9927,0x12c211211000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9484,108_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op1,kernel_graph0,{9703-9708-9715-9911-9925},9925,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9705,9921,0x12c219211400,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9485,110_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/Cast-op0,kernel_graph0,{9708-9918},9918,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9707,9903,0x12c21e811600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9485,111_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,kernel_graph0,{9710-9721-9901},9901,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9708,9710,0x12c19400e600,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9485,111_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9712,9936,0x12c229411800,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9485,113_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/Cast-op0,kernel_graph0,{9715-9933},9933,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9714,9909,0x12c22ea11a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9486,114_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,kernel_graph0,{9718-9906},9906,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9715,9717,0x12c19400e600,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9486,114_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9718,9899,0x12c239611c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9487,115_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/SiLU-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/SiLU-op0,kernel_graph0,{9719-9721-9897},9897,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9721,9886,0x12c244211e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9488,117_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/Mul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/Mul-op0,kernel_graph0,{9722-9727-9884},9884,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9724,9894,0x12c24ee12000,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9491,119_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/Cast-op0,kernel_graph0,{9727-9892},9892,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9726,9733,0x12c254412200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9491,120_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,kernel_graph0,{9729-9731},9731,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9727,9729,0x12c19400e600,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9491,120_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9731,9865,0x12c258412400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9492,122_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/Add-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/Add-op1,kernel_graph0,{9734-9863},9863,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9734,9740,0x12c25c412600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9493,123_Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Cast-op0,kernel_graph0,{9738},9738,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9736,9744,0x12c264412800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9495,124_Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{9742},9742,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9737,9872,0x12c187939400,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9495,124_Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{9868},9868,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9738,9741,0x12c19400e600,0,DefaultEnhancedAscendMemoryPool,16777728,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9495,124_Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9742,9848,0x12c254412200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9496,125_Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Cast-op1,kernel_graph0,{9744-9749-9846},9846,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9746,9857,0x12c25c412600,0,DefaultEnhancedAscendMemoryPool,262144512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9499,127_Default/network-LlamaForCausalLM/lm_head-Linear/Cast-op0,Default/network-LlamaForCausalLM/lm_head-Linear/Cast-op0,kernel_graph0,{9749-9854},9854,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9748,9754,0x12c26be12800,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9500,128_Default/network-LlamaForCausalLM/lm_head-Linear/MatMul-op0,Default/network-LlamaForCausalLM/lm_head-Linear/MatMul-op0,kernel_graph0,{9752},9752,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9749,9751,0x12c19400e600,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9500,128_Default/network-LlamaForCausalLM/lm_head-Linear/MatMul-op0,Default/network-LlamaForCausalLM/lm_head-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9752,9761,0x12c28b212a00,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9500,129_Default/network-LlamaForCausalLM/Cast-op0,Default/network-LlamaForCausalLM/Cast-op0,kernel_graph0,{9756-9759},9759,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9755,9762,0x12c187941600,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9501,130_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/ReduceMax-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/ReduceMax-op0,kernel_graph0,{9759},9759,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9756,9758,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9501,130_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/ReduceMax-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/ReduceMax-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9759,9795,0x12c2c9a12c00,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9502,131_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Sub-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Sub-op0,kernel_graph0,{9765-9793},9793,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9763,9776,0x12c180009200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9502,132_Default/Mul-op0,Default/Mul-op0,kernel_graph0,{9774},9774,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9765,9773,0x12c26be12800,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9504,133_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Exp-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Exp-op0,kernel_graph0,{9767},9767,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9767,9792,0x12c187941600,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9505,134_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/ReduceSum-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/ReduceSum-op0,kernel_graph0,{9790},9790,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9769,9771,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,9769,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9770,9772,0x12c180009c00,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,9770,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9774,9780,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9505,135_Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_RealDiv/RealDiv-op0,Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_RealDiv/RealDiv-op0,kernel_graph0,{9778},9778,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9777,9785,0x12c180009200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9507,136_Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_RealDiv/SumExt-op0,Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_RealDiv/SumExt-op0,kernel_graph0,{9781-9783},9783,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9778,9781,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9507,136_Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_RealDiv/SumExt-op0,Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_RealDiv/SumExt-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9783,9788,0x12c187949800,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9508,138_Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_ReduceSum/Tile-op0,Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_ReduceSum/Tile-op0,kernel_graph0,{9786},9786,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9786,9812,0x12c187951a00,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9510,139_Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_Mul/Mul-op0,Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_Mul/Mul-op0,kernel_graph0,{9788-9809},9809,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9790,9796,0x12c187949800,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9512,141_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Log-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Log-op0,kernel_graph0,{9793},9793,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9793,9818,0x12c26be12800,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9512,142_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Sub-op1,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Sub-op1,kernel_graph0,{9804-9816},9816,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9797,9803,0x12c187941600,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9514,143_Default/network-LlamaForCausalLM/StridedSlice-op1,Default/network-LlamaForCausalLM/StridedSlice-op1,kernel_graph0,{9799-9801},9801,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9801,9819,0x12c2aa612a00,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9515,145_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/OneHot-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/OneHot-op0,kernel_graph0,{9806-9816},9816,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9804,9808,0x12c2e8e12c00,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9517,146_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Exp-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Exp-op0,kernel_graph0,{9806},9806,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9806,9811,0x12c327612e00,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9518,147_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Sub-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Sub-op0,kernel_graph0,{9809},9809,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9809,9815,0x12c2e8e12c00,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9518,148_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Mul-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Mul-op0,kernel_graph0,{9813},9813,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9813,9856,0x12c327612e00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9520,149_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op0,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op0,kernel_graph0,{9846-9854},9854,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9816,9822,0x12c2e8e12c00,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9521,150_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Mul-op1,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Mul-op1,kernel_graph0,{9820},9820,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9820,9829,0x12c26be12800,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9522,151_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Neg-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Neg-op0,kernel_graph0,{9823},9823,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9823,9832,0x12c187941600,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9522,152_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/ReduceSum-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/ReduceSum-op0,kernel_graph0,{9830},9830,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9825,9827,0x12c180009200,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,9825,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9826,9828,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,9826,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9830,9840,0x12c187949800,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9528,153_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Mul-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Mul-op0,kernel_graph0,{9834},9834,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9834,9843,0x12c180009200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9529,154_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/ReduceSum-op1,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/ReduceSum-op1,kernel_graph0,{9841},9841,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9836,9838,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,9836,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9837,9839,0x12c180009c00,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,9837,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9841,11677,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9529,155_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/RealDiv-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/RealDiv-op0,kernel_graph0,{11675},11675,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9845,9852,0x12c346a13000,0,DefaultEnhancedAscendMemoryPool,262144512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9529,156_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op29,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op29,kernel_graph0,{9850},9850,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9846,9849,0x12c19400e600,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9529,156_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op29,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op29,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9850,10431,0x12c26be12800,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9532,157_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op29,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op29,kernel_graph0,{10429},10429,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9853,9862,0x12c254412200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9533,158_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op0,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op0,kernel_graph0,{9858-9860},9860,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9854,9858,0x12c346a13000,0,DefaultEnhancedAscendMemoryPool,155190272,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9533,158_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op0,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9860,9870,0x12c25c412600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9534,160_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op1,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op1,kernel_graph0,{9868},9868,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9863,9871,0x12c28b212a00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9534,161_Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Cast-op2,kernel_graph0,{9868},9868,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9866,9876,0x12c254412200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9535,162_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op0,kernel_graph0,{9874},9874,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9867,10446,0x12c18000ba00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9535,162_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op0,kernel_graph0,{10444},10444,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9868,9873,0x12c19400e600,0,DefaultEnhancedAscendMemoryPool,16778752,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9535,162_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9874,9963,0x12c25c412400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9536,163_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op0,kernel_graph0,{9878-9884-9892-9960},9960,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9877,9882,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9537,164_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Mul-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Mul-op0,kernel_graph0,{9880},9880,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9880,10646,0x12c180009200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9539,166_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Maximum-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Maximum-op0,kernel_graph0,{10644},10644,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9883,9890,0x12c254412200,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9540,167_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op27,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op27,kernel_graph0,{9888},9888,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9884,9887,0x12c19400e600,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9540,167_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op27,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op27,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9888,10425,0x12c244211e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9541,168_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op27,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op27,kernel_graph0,{10423},10423,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9891,9904,0x12c260412600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9541,169_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op1,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op1,kernel_graph0,{9895-9897-9901},9901,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9892,9895,0x12c19400e600,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9541,169_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op1,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9897,9920,0x12c24ee12000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9543,171_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op0,kernel_graph0,{9899-9911-9918},9918,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9901,9908,0x12c239611c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9544,173_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op1,kernel_graph0,{9904-9906},9906,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9906,9935,0x12c21e811600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9547,175_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/Grad_SiLU/SiLUGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/Grad_SiLU/SiLUGrad-op0,kernel_graph0,{9925-9933},9933,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9910,9916,0x12c260412600,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9547,176_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op28,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op28,kernel_graph0,{9914},9914,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9911,9913,0x12c19400e600,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9547,176_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op28,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op28,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9914,10428,0x12c22ea11a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9550,177_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op28,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op28,kernel_graph0,{10426},10426,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9917,9941,0x12c239611c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9551,178_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op2,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op2,kernel_graph0,{9922-9939},9939,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9918,9922,0x12c19400e600,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9551,178_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op2,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9924,9931,0x12c219211400,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9553,180_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op26,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op26,kernel_graph0,{9929},9929,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9925,9928,0x12c19400e600,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9553,180_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op26,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op26,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9929,10422,0x12c260412600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9554,181_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op26,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op26,kernel_graph0,{10420},10420,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9932,9942,0x12c211211000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9555,182_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op3,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op3,kernel_graph0,{9937-9939},9939,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9933,9937,0x12c19400e600,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9555,182_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op3,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op3,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9939,9945,0x12c23d611e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9557,184_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/AddN-op0,kernel_graph0,{9943},9943,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9943,9953,0x12c24ee12000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9558,185_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op2,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op2,kernel_graph0,{9951},9951,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9946,9954,0x12c239611c00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9559,186_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op2,kernel_graph0,{9951},9951,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9949,9959,0x12c211211000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9559,187_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op1,kernel_graph0,{9957},9957,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9950,10440,0x12c18000fc00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9559,187_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op1,kernel_graph0,{10438},10438,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9951,9956,0x12c19400e600,0,DefaultEnhancedAscendMemoryPool,16778752,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9559,187_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9957,9962,0x12c239611c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9561,188_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op1,kernel_graph0,{9960},9960,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9960,10127,0x12c23d611e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9562,189_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/AddN-op0,kernel_graph0,{9965-9971-9979-10124},10124,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9964,9969,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9562,190_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Sub-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Sub-op0,kernel_graph0,{9967},9967,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9967,10669,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9564,192_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/LessEqual-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/LessEqual-op0,kernel_graph0,{10649-10667},10667,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9970,9977,0x12c241612000,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9565,193_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op25,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op25,kernel_graph0,{9975},9975,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9971,9974,0x12c19400e600,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9565,193_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op25,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op25,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9975,10419,0x12c20b210c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9566,194_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op25,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op25,kernel_graph0,{10417},10417,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9978,9987,0x12c239611c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9567,195_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op4,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op4,kernel_graph0,{9982-9985},9985,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9979,9982,0x12c19400e600,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9567,195_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op4,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op4,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9984,9998,0x12c24ee12000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9569,197_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op0,kernel_graph0,{9993},9993,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9985,9988,0x12c180009a00,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9569,197_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9989,10010,0x12c239611c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9570,198_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,kernel_graph0,{10008},10008,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9990,10007,0x12c252e12200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9570,198_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,kernel_graph0,{10005},10005,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9991,10014,0x12c256e12400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9570,198_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,kernel_graph0,{10012},10012,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9992,10003,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9570,198_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 9993,10004,0x12c20f210e00,0,DefaultEnhancedAscendMemoryPool,515969024,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9570,198_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10005,10021,0x12c1f9210200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9571,199_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op5,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op5,kernel_graph0,{10017-10019},10019,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10008,10042,0x12c203210800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9572,200_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op3,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op3,kernel_graph0,{10038-10040},10040,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10011,10069,0x12c239611c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9572,201_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op3,kernel_graph0,{10015-10060-10067},10067,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10012,10015,0x12c180009a00,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9572,201_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op3,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10017,10027,0x12c24ee12000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9572,203_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op4,kernel_graph0,{10025},10025,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10019,10024,0x12c256e12200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9573,204_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op5,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op5,kernel_graph0,{10022},10022,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10022,10028,0x12c1f9210200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9574,205_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op1,kernel_graph0,{10025},10025,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10025,10031,0x12c256e12200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9575,206_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op1,kernel_graph0,{10029},10029,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10029,10035,0x12c24ee12000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9575,207_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op6,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op6,kernel_graph0,{10033},10033,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10032,10083,0x12c1f9210200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9575,208_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op2,kernel_graph0,{10036-10074-10081},10081,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10033,10036,0x12c180009a00,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9575,208_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10038,10048,0x12c24ee12000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9577,210_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op2,kernel_graph0,{10046},10046,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10040,10045,0x12c256e12200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9578,211_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op3,kernel_graph0,{10043},10043,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10043,10049,0x12c203210800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9578,212_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op0,kernel_graph0,{10046},10046,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10046,10052,0x12c256e12200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9580,213_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op0,kernel_graph0,{10050},10050,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10050,10056,0x12c1fd210400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9581,214_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op4,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op4,kernel_graph0,{10054},10054,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10053,10098,0x12c203210800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9581,215_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op1,kernel_graph0,{10057-10088-10096},10096,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10054,10057,0x12c180009a00,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9581,215_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10059,10065,0x12c241612000,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9583,217_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op24,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op24,kernel_graph0,{10063},10063,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10060,10062,0x12c1ae00f200,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9583,217_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op24,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op24,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10063,10416,0x12c1fd210400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9584,218_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op24,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op24,kernel_graph0,{10414},10414,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10066,10106,0x12c207210a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9585,219_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op7,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op7,kernel_graph0,{10071-10102},10102,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10067,10071,0x12c1ae00f200,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9585,219_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op7,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op7,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10073,10079,0x12c201210600,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9587,221_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op23,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op23,kernel_graph0,{10077},10077,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10074,10076,0x12c1ae00f200,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9587,221_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op23,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op23,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10077,10413,0x12c239611c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9588,222_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op23,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op23,kernel_graph0,{10411},10411,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10080,10105,0x12c24ee12000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9588,223_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op6,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op6,kernel_graph0,{10085-10102},10102,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10081,10085,0x12c1ae00f200,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9588,223_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op6,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op6,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10087,10094,0x12c201210600,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9591,225_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op22,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op22,kernel_graph0,{10092},10092,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10088,10091,0x12c19400e600,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9591,225_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op22,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op22,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10092,10410,0x12c1ef20fc00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9592,226_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op22,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op22,kernel_graph0,{10408},10408,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10095,10104,0x12c1f9210200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9592,227_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op5,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op5,kernel_graph0,{10100-10102},10102,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10096,10100,0x12c19400e600,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9592,227_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op5,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op5,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10102,10109,0x12c201210600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9596,229_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/AddN-op0,kernel_graph0,{10107},10107,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10107,10117,0x12c24ee12000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9596,230_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op7,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op7,kernel_graph0,{10115},10115,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10110,10118,0x12c256e12200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9597,231_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op2,kernel_graph0,{10115},10115,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10113,10123,0x12c1f320fe00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9598,232_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op2,kernel_graph0,{10121},10121,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10114,10443,0x12c187931200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9598,232_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op2,kernel_graph0,{10441},10441,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10115,10120,0x12c25ee12400,0,DefaultEnhancedAscendMemoryPool,16778752,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9598,232_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10121,10126,0x12c201210600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9599,233_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op2,kernel_graph0,{10124},10124,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10124,10209,0x12c205210800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9600,234_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/AddN-op0,kernel_graph0,{10127-10130-10138-10206},10206,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10129,10136,0x12c23d611e00,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9602,236_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op20,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op20,kernel_graph0,{10134},10134,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10130,10133,0x12c242c12000,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9602,236_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op20,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op20,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10134,10404,0x12c1df00f800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9603,237_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op20,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op20,kernel_graph0,{10402},10402,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10137,10150,0x12c24ee12000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9604,238_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op8,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op8,kernel_graph0,{10141-10143-10147},10147,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10138,10141,0x12c19400e600,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9604,238_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op8,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op8,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10143,10166,0x12c20f210e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9607,240_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op6,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op6,kernel_graph0,{10145-10157-10164},10164,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10147,10154,0x12c1d440f600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9608,242_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op7,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op7,kernel_graph0,{10150-10152},10152,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10152,10181,0x12c1b960f000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9612,244_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/Grad_SiLU/SiLUGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/Grad_SiLU/SiLUGrad-op1,kernel_graph0,{10171-10179},10179,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10156,10162,0x12c1e9c0fa00,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9612,245_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op21,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op21,kernel_graph0,{10160},10160,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10157,10159,0x12c19400e600,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9612,245_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op21,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op21,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10160,10407,0x12c24ee12000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9614,246_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op21,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op21,kernel_graph0,{10405},10405,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10163,10187,0x12c201210600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9614,247_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op9,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op9,kernel_graph0,{10168-10185},10185,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10164,10168,0x12c19400e600,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9614,247_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op9,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op9,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10170,10177,0x12c1b400ee00,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9616,249_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op19,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op19,kernel_graph0,{10175},10175,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10171,10174,0x12c19400e600,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9616,249_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op19,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op19,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10175,10401,0x12c1c980f400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9617,250_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op19,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op19,kernel_graph0,{10399},10399,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10178,10188,0x12c1a400ec00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9618,251_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op10,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op10,kernel_graph0,{10183-10185},10185,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10179,10183,0x12c19400e600,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9618,251_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op10,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op10,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10185,10191,0x12c1e9c0fa00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9620,253_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/AddN-op0,kernel_graph0,{10189},10189,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10189,10199,0x12c1f320fe00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9622,254_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op8,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op8,kernel_graph0,{10197},10197,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10192,10200,0x12c1d440f600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9623,255_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op2,kernel_graph0,{10197},10197,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10195,10205,0x12c1a400ec00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9623,256_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op3,kernel_graph0,{10203},10203,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10196,10434,0x12c187929000,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9623,256_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op3,kernel_graph0,{10432},10432,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10197,10202,0x12c19400e600,0,DefaultEnhancedAscendMemoryPool,16778752,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9623,256_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op3,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10203,10208,0x12c201210600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9626,257_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op3,kernel_graph0,{10206},10206,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10206,10371,0x12c1e9c0fa00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9627,258_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/AddN-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/AddN-op1,kernel_graph0,{10209-10212-10220-10368},10368,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10211,10218,0x12c259a12200,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9629,260_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op18,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op18,kernel_graph0,{10216},10216,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10212,10215,0x12c1edc0fc00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9629,260_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op18,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op18,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10216,10398,0x12c25ba12400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9629,261_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op18,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op18,kernel_graph0,{10396},10396,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10219,10228,0x12c23d611e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9630,262_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op11,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op11,kernel_graph0,{10223-10226},10226,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10220,10223,0x12c1edc0fc00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9630,262_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op11,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op11,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10225,10239,0x12c1f320fe00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9631,264_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op4,kernel_graph0,{10234},10234,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10226,10229,0x12c180009a00,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9631,264_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op4,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10230,10252,0x12c1f7210000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9632,265_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,kernel_graph0,{10250},10250,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10231,10249,0x12c23d611e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9632,265_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,kernel_graph0,{10247},10247,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10232,10256,0x12c201210600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9632,265_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,kernel_graph0,{10254},10254,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10233,10245,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9632,265_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10234,10246,0x12c20f210e00,0,DefaultEnhancedAscendMemoryPool,515969024,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9632,265_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10247,10263,0x12c18e00e200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9633,266_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op11,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op11,kernel_graph0,{10259-10261},10261,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10250,10286,0x12c1d440f600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9634,267_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op9,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op9,kernel_graph0,{10280-10283},10283,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10253,10313,0x12c205210800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9635,268_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op7,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op7,kernel_graph0,{10257-10304-10311},10311,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10254,10257,0x12c180009a00,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9635,268_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op7,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op7,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10259,10269,0x12c1f320fe00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9637,270_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op10,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op10,kernel_graph0,{10267},10267,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10261,10266,0x12c20f210e00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9638,271_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op11,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op11,kernel_graph0,{10264},10264,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10264,10270,0x12c18e00e200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9638,272_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op3,kernel_graph0,{10267},10267,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10267,10273,0x12c20f210e00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9641,273_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op1,kernel_graph0,{10271},10271,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10271,10277,0x12c201210600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9641,274_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op12,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op12,kernel_graph0,{10275},10275,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10274,10327,0x12c23d611e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9642,275_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op6,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op6,kernel_graph0,{10278-10318-10325},10325,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10275,10278,0x12c180009a00,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9642,275_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op6,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op6,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10280,10292,0x12c18e00e200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9644,277_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op8,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op8,kernel_graph0,{10290},10290,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10283,10289,0x12c1f320fe00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9645,278_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op9,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op9,kernel_graph0,{10287},10287,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10287,10293,0x12c1d440f600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9645,279_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op2,kernel_graph0,{10290},10290,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10290,10296,0x12c1f320fe00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9646,280_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op0,kernel_graph0,{10294},10294,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10294,10300,0x12c201210600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9647,281_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op10,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op10,kernel_graph0,{10298},10298,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10297,10342,0x12c18e00e200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9647,282_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op5,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op5,kernel_graph0,{10301-10332-10340},10340,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10298,10301,0x12c180009a00,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9647,282_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op5,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op5,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10303,10309,0x12c209210a00,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9650,284_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op17,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op17,kernel_graph0,{10307},10307,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10304,10306,0x12c1edc0fc00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9650,284_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op17,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op17,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10307,10395,0x12c201210600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9650,285_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op17,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op17,kernel_graph0,{10393},10393,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10310,10350,0x12c1f320fe00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9652,286_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op14,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op14,kernel_graph0,{10315-10346},10346,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10311,10315,0x12c1edc0fc00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9652,286_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op14,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op14,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10317,10323,0x12c259a12200,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9653,288_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op16,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op16,kernel_graph0,{10321},10321,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10318,10320,0x12c1edc0fc00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9653,288_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op16,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op16,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10321,10392,0x12c1f7210000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9655,289_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op16,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op16,kernel_graph0,{10390},10390,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10324,10349,0x12c205210800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9656,290_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op13,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op13,kernel_graph0,{10329-10346},10346,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10325,10329,0x12c1edc0fc00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9656,290_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op13,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op13,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10331,10338,0x12c1fb210200,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9658,292_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op15,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op15,kernel_graph0,{10336},10336,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10332,10335,0x12c1edc0fc00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9658,292_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op15,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op15,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10336,10389,0x12c23d611e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9659,293_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op15,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op15,kernel_graph0,{10387},10387,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10339,10348,0x12c1800a0200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9660,294_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op12,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op12,kernel_graph0,{10344-10346},10346,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10340,10344,0x12c1edc0fc00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9660,294_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op12,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op12,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10346,10353,0x12c1d440f600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9662,296_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/AddN-op0,kernel_graph0,{10351},10351,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10351,10361,0x12c20f210e00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9663,297_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op13,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op13,kernel_graph0,{10359},10359,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10354,10362,0x12c1d440f600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9663,298_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op2,kernel_graph0,{10359},10359,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10357,10367,0x12c217211000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9664,299_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op4,kernel_graph0,{10365},10365,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10358,10437,0x12c180013e00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9664,299_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op4,kernel_graph0,{10435},10435,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10359,10364,0x12c1edc0fc00,0,DefaultEnhancedAscendMemoryPool,16778752,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9664,299_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op4,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10365,10370,0x12c1f320fe00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9665,300_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op4,kernel_graph0,{10368},10368,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10368,10374,0x12c205210800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9667,301_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/AddN-op0,kernel_graph0,{10372},10372,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10372,10377,0x12c1d440f600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9668,302_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op14,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op14,kernel_graph0,{10375},10375,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10375,10386,0x12c20f210e00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9668,303_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/tok_embeddings-LlamaEmbedding/Grad_Gather/UnsortedSegmentSum-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/tok_embeddings-LlamaEmbedding/Grad_Gather/UnsortedSegmentSum-op0,kernel_graph0,{10384},10384,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10379,10383,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9672,304_Default/Mul-op1,Default/Mul-op1,kernel_graph0,{10381},10381,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10381,10447,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9675,305_Default/Reciprocal-op0,Default/Reciprocal-op0,kernel_graph0,{10384-10387-10390-10393-10396-10399-10402-10405-10408-10411-10414-10417-10420-10423-10426-10429-10432-10435-10438-10441-10444},10444,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10384,10682,0x12c187935400,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9675,306_Default/Mul-op2,Default/Mul-op2,kernel_graph0,{10448-10608-10680},10680,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10387,10685,0x12c1f320fe00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9676,307_Default/Mul-op3,Default/Mul-op3,kernel_graph0,{10456-10608-10683},10683,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10390,10688,0x12c1e9c0fa00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9676,308_Default/Mul-op4,Default/Mul-op4,kernel_graph0,{10464-10608-10686},10686,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10393,10691,0x12c1f7210000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9676,309_Default/Mul-op5,Default/Mul-op5,kernel_graph0,{10472-10608-10689},10689,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10396,10694,0x12c23d611e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9679,310_Default/Mul-op6,Default/Mul-op6,kernel_graph0,{10480-10608-10692},10692,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10399,10697,0x12c1d440f600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9680,311_Default/Mul-op7,Default/Mul-op7,kernel_graph0,{10488-10608-10695},10695,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10402,10700,0x12c20f210e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9680,312_Default/Mul-op8,Default/Mul-op8,kernel_graph0,{10496-10608-10698},10698,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10405,10703,0x12c1df00f800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9681,313_Default/Mul-op9,Default/Mul-op9,kernel_graph0,{10504-10608-10701},10701,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10408,10706,0x12c180018000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9681,314_Default/Mul-op10,Default/Mul-op10,kernel_graph0,{10512-10608-10704},10704,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10411,10709,0x12c1edc0fc00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9682,315_Default/Mul-op11,Default/Mul-op11,kernel_graph0,{10520-10608-10707},10707,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10414,10712,0x12c239611c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9683,316_Default/Mul-op12,Default/Mul-op12,kernel_graph0,{10528-10608-10710},10710,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10417,10715,0x12c1fb210200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9684,317_Default/Mul-op13,Default/Mul-op13,kernel_graph0,{10536-10608-10713},10713,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10420,10718,0x12c1ff210400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9684,318_Default/Mul-op14,Default/Mul-op14,kernel_graph0,{10544-10608-10716},10716,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10423,10721,0x12c219e11000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9686,319_Default/Mul-op15,Default/Mul-op15,kernel_graph0,{10552-10608-10719},10719,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10426,10724,0x12c241612000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9686,320_Default/Mul-op16,Default/Mul-op16,kernel_graph0,{10560-10608-10722},10722,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10429,10727,0x12c24c212200,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9687,321_Default/Mul-op17,Default/Mul-op17,kernel_graph0,{10568-10608-10725},10725,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10432,10730,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9687,322_Default/Mul-op18,Default/Mul-op18,kernel_graph0,{10576-10608-10728},10728,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10435,10733,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9688,323_Default/Mul-op19,Default/Mul-op19,kernel_graph0,{10584-10608-10731},10731,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10438,10736,0x12c180013e00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9689,324_Default/Mul-op20,Default/Mul-op20,kernel_graph0,{10592-10608-10734},10734,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10441,10739,0x12c18000fc00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9690,325_Default/Mul-op21,Default/Mul-op21,kernel_graph0,{10600-10608-10737},10737,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10444,10742,0x12c1f1c0fe00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9691,326_Default/Mul-op22,Default/Mul-op22,kernel_graph0,{10608-10610-10740},10740,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10448,10453,0x12c1a6d35600,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9691,327_Default/clip_grad_norm-ClipGradNorm/Square-op0,Default/clip_grad_norm-ClipGradNorm/Square-op0,kernel_graph0,{10451},10451,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10450,10620,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9693,328_Default/clip_grad_norm-ClipGradNorm/SumExt-op0,Default/clip_grad_norm-ClipGradNorm/SumExt-op0,kernel_graph0,{10454-10618},10618,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10451,10454,0x12c18000ba00,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9693,328_Default/clip_grad_norm-ClipGradNorm/SumExt-op0,Default/clip_grad_norm-ClipGradNorm/SumExt-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10456,10461,0x12c209e10600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9694,330_Default/clip_grad_norm-ClipGradNorm/Square-op1,Default/clip_grad_norm-ClipGradNorm/Square-op1,kernel_graph0,{10459},10459,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10458,10621,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9697,331_Default/clip_grad_norm-ClipGradNorm/SumExt-op1,Default/clip_grad_norm-ClipGradNorm/SumExt-op1,kernel_graph0,{10462-10618},10618,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10459,10462,0x12c18000ba00,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9697,331_Default/clip_grad_norm-ClipGradNorm/SumExt-op1,Default/clip_grad_norm-ClipGradNorm/SumExt-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10464,10469,0x12c209e10600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9698,333_Default/clip_grad_norm-ClipGradNorm/Square-op2,Default/clip_grad_norm-ClipGradNorm/Square-op2,kernel_graph0,{10467},10467,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10466,10622,0x12c180009a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9701,334_Default/clip_grad_norm-ClipGradNorm/SumExt-op2,Default/clip_grad_norm-ClipGradNorm/SumExt-op2,kernel_graph0,{10470-10618},10618,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10467,10470,0x12c18000ba00,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9701,334_Default/clip_grad_norm-ClipGradNorm/SumExt-op2,Default/clip_grad_norm-ClipGradNorm/SumExt-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10472,10477,0x12c209e10600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9702,336_Default/clip_grad_norm-ClipGradNorm/Square-op3,Default/clip_grad_norm-ClipGradNorm/Square-op3,kernel_graph0,{10475},10475,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10474,10623,0x12c180009c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9703,337_Default/clip_grad_norm-ClipGradNorm/SumExt-op3,Default/clip_grad_norm-ClipGradNorm/SumExt-op3,kernel_graph0,{10478-10618},10618,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10475,10478,0x12c18000ba00,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9703,337_Default/clip_grad_norm-ClipGradNorm/SumExt-op3,Default/clip_grad_norm-ClipGradNorm/SumExt-op3,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10480,10485,0x12c209e10600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9705,339_Default/clip_grad_norm-ClipGradNorm/Square-op4,Default/clip_grad_norm-ClipGradNorm/Square-op4,kernel_graph0,{10483},10483,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10482,10624,0x12c180009e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9706,340_Default/clip_grad_norm-ClipGradNorm/SumExt-op4,Default/clip_grad_norm-ClipGradNorm/SumExt-op4,kernel_graph0,{10486-10618},10618,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10483,10486,0x12c18000ba00,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9706,340_Default/clip_grad_norm-ClipGradNorm/SumExt-op4,Default/clip_grad_norm-ClipGradNorm/SumExt-op4,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10488,10493,0x12c224a11200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9708,342_Default/clip_grad_norm-ClipGradNorm/Square-op5,Default/clip_grad_norm-ClipGradNorm/Square-op5,kernel_graph0,{10491},10491,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10490,10625,0x12c18000a000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9709,343_Default/clip_grad_norm-ClipGradNorm/SumExt-op5,Default/clip_grad_norm-ClipGradNorm/SumExt-op5,kernel_graph0,{10494-10618},10618,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10491,10494,0x12c18000ba00,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9709,343_Default/clip_grad_norm-ClipGradNorm/SumExt-op5,Default/clip_grad_norm-ClipGradNorm/SumExt-op5,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10496,10501,0x12c224a11200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9710,345_Default/clip_grad_norm-ClipGradNorm/Square-op6,Default/clip_grad_norm-ClipGradNorm/Square-op6,kernel_graph0,{10499},10499,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10498,10626,0x12c18000a200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9711,346_Default/clip_grad_norm-ClipGradNorm/SumExt-op6,Default/clip_grad_norm-ClipGradNorm/SumExt-op6,kernel_graph0,{10502-10618},10618,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10499,10502,0x12c18000ba00,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9711,346_Default/clip_grad_norm-ClipGradNorm/SumExt-op6,Default/clip_grad_norm-ClipGradNorm/SumExt-op6,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10504,10509,0x12c224a11200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9713,348_Default/clip_grad_norm-ClipGradNorm/Square-op7,Default/clip_grad_norm-ClipGradNorm/Square-op7,kernel_graph0,{10507},10507,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10506,10627,0x12c18000a400,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9713,349_Default/clip_grad_norm-ClipGradNorm/SumExt-op7,Default/clip_grad_norm-ClipGradNorm/SumExt-op7,kernel_graph0,{10510-10618},10618,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10507,10510,0x12c18000ba00,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9713,349_Default/clip_grad_norm-ClipGradNorm/SumExt-op7,Default/clip_grad_norm-ClipGradNorm/SumExt-op7,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10512,10517,0x12c209e10600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9716,351_Default/clip_grad_norm-ClipGradNorm/Square-op8,Default/clip_grad_norm-ClipGradNorm/Square-op8,kernel_graph0,{10515},10515,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10514,10628,0x12c18000a600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9716,352_Default/clip_grad_norm-ClipGradNorm/SumExt-op8,Default/clip_grad_norm-ClipGradNorm/SumExt-op8,kernel_graph0,{10518-10618},10618,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10515,10518,0x12c18000ba00,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9716,352_Default/clip_grad_norm-ClipGradNorm/SumExt-op8,Default/clip_grad_norm-ClipGradNorm/SumExt-op8,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10520,10525,0x12c209e10600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9719,354_Default/clip_grad_norm-ClipGradNorm/Square-op9,Default/clip_grad_norm-ClipGradNorm/Square-op9,kernel_graph0,{10523},10523,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10522,10629,0x12c18000a800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9719,355_Default/clip_grad_norm-ClipGradNorm/SumExt-op9,Default/clip_grad_norm-ClipGradNorm/SumExt-op9,kernel_graph0,{10526-10618},10618,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10523,10526,0x12c18000ba00,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9719,355_Default/clip_grad_norm-ClipGradNorm/SumExt-op9,Default/clip_grad_norm-ClipGradNorm/SumExt-op9,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10528,10533,0x12c209e10600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9720,357_Default/clip_grad_norm-ClipGradNorm/Square-op10,Default/clip_grad_norm-ClipGradNorm/Square-op10,kernel_graph0,{10531},10531,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10530,10630,0x12c18000aa00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9721,358_Default/clip_grad_norm-ClipGradNorm/SumExt-op10,Default/clip_grad_norm-ClipGradNorm/SumExt-op10,kernel_graph0,{10534-10618},10618,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10531,10534,0x12c18000ba00,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9721,358_Default/clip_grad_norm-ClipGradNorm/SumExt-op10,Default/clip_grad_norm-ClipGradNorm/SumExt-op10,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10536,10541,0x12c209e10600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9722,360_Default/clip_grad_norm-ClipGradNorm/Square-op11,Default/clip_grad_norm-ClipGradNorm/Square-op11,kernel_graph0,{10539},10539,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10538,10631,0x12c18000ac00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9722,361_Default/clip_grad_norm-ClipGradNorm/SumExt-op11,Default/clip_grad_norm-ClipGradNorm/SumExt-op11,kernel_graph0,{10542-10618},10618,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10539,10542,0x12c18000ba00,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9722,361_Default/clip_grad_norm-ClipGradNorm/SumExt-op11,Default/clip_grad_norm-ClipGradNorm/SumExt-op11,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10544,10549,0x12c224a11200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9724,363_Default/clip_grad_norm-ClipGradNorm/Square-op12,Default/clip_grad_norm-ClipGradNorm/Square-op12,kernel_graph0,{10547},10547,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10546,10632,0x12c18000ae00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9725,364_Default/clip_grad_norm-ClipGradNorm/SumExt-op12,Default/clip_grad_norm-ClipGradNorm/SumExt-op12,kernel_graph0,{10550-10618},10618,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10547,10550,0x12c18000ba00,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9725,364_Default/clip_grad_norm-ClipGradNorm/SumExt-op12,Default/clip_grad_norm-ClipGradNorm/SumExt-op12,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10552,10557,0x12c224a11200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9727,366_Default/clip_grad_norm-ClipGradNorm/Square-op13,Default/clip_grad_norm-ClipGradNorm/Square-op13,kernel_graph0,{10555},10555,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10554,10633,0x12c18000b000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9728,367_Default/clip_grad_norm-ClipGradNorm/SumExt-op13,Default/clip_grad_norm-ClipGradNorm/SumExt-op13,kernel_graph0,{10558-10618},10618,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10555,10558,0x12c18000ba00,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9728,367_Default/clip_grad_norm-ClipGradNorm/SumExt-op13,Default/clip_grad_norm-ClipGradNorm/SumExt-op13,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10560,10565,0x12c224a11200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9729,369_Default/clip_grad_norm-ClipGradNorm/Square-op14,Default/clip_grad_norm-ClipGradNorm/Square-op14,kernel_graph0,{10563},10563,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10562,10634,0x12c18000b200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9730,370_Default/clip_grad_norm-ClipGradNorm/SumExt-op14,Default/clip_grad_norm-ClipGradNorm/SumExt-op14,kernel_graph0,{10566-10618},10618,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10563,10566,0x12c18000ba00,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9730,370_Default/clip_grad_norm-ClipGradNorm/SumExt-op14,Default/clip_grad_norm-ClipGradNorm/SumExt-op14,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10568,10573,0x12c1a6d35600,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9732,372_Default/clip_grad_norm-ClipGradNorm/Square-op15,Default/clip_grad_norm-ClipGradNorm/Square-op15,kernel_graph0,{10571},10571,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10570,10635,0x12c18000b400,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9732,373_Default/clip_grad_norm-ClipGradNorm/SumExt-op15,Default/clip_grad_norm-ClipGradNorm/SumExt-op15,kernel_graph0,{10574-10618},10618,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10571,10574,0x12c18000ba00,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9732,373_Default/clip_grad_norm-ClipGradNorm/SumExt-op15,Default/clip_grad_norm-ClipGradNorm/SumExt-op15,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10576,10581,0x12c18000ba00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9735,375_Default/clip_grad_norm-ClipGradNorm/Square-op16,Default/clip_grad_norm-ClipGradNorm/Square-op16,kernel_graph0,{10579},10579,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10578,10636,0x12c18000b600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9735,376_Default/clip_grad_norm-ClipGradNorm/SumExt-op16,Default/clip_grad_norm-ClipGradNorm/SumExt-op16,kernel_graph0,{10582-10618},10618,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10579,10582,0x12c1f1c14000,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9735,376_Default/clip_grad_norm-ClipGradNorm/SumExt-op16,Default/clip_grad_norm-ClipGradNorm/SumExt-op16,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10584,10589,0x12c18000ba00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9739,378_Default/clip_grad_norm-ClipGradNorm/Square-op17,Default/clip_grad_norm-ClipGradNorm/Square-op17,kernel_graph0,{10587},10587,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10586,10637,0x12c1f1c14000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9739,379_Default/clip_grad_norm-ClipGradNorm/SumExt-op17,Default/clip_grad_norm-ClipGradNorm/SumExt-op17,kernel_graph0,{10590-10618},10618,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10587,10590,0x12c1f1c14200,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9739,379_Default/clip_grad_norm-ClipGradNorm/SumExt-op17,Default/clip_grad_norm-ClipGradNorm/SumExt-op17,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10592,10597,0x12c18000ba00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9742,381_Default/clip_grad_norm-ClipGradNorm/Square-op18,Default/clip_grad_norm-ClipGradNorm/Square-op18,kernel_graph0,{10595},10595,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10594,10638,0x12c1f1c14200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9743,382_Default/clip_grad_norm-ClipGradNorm/SumExt-op18,Default/clip_grad_norm-ClipGradNorm/SumExt-op18,kernel_graph0,{10598-10618},10618,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10595,10598,0x12c1f1c14400,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9743,382_Default/clip_grad_norm-ClipGradNorm/SumExt-op18,Default/clip_grad_norm-ClipGradNorm/SumExt-op18,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10600,10605,0x12c18000ba00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9744,384_Default/clip_grad_norm-ClipGradNorm/Square-op19,Default/clip_grad_norm-ClipGradNorm/Square-op19,kernel_graph0,{10603},10603,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10602,10639,0x12c1f1c14400,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9745,385_Default/clip_grad_norm-ClipGradNorm/SumExt-op19,Default/clip_grad_norm-ClipGradNorm/SumExt-op19,kernel_graph0,{10606-10618},10618,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10603,10606,0x12c1f1c14600,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9745,385_Default/clip_grad_norm-ClipGradNorm/SumExt-op19,Default/clip_grad_norm-ClipGradNorm/SumExt-op19,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10608,13973,0x12c18000ba00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9747,387_Default/AllFinite-op0,Default/AllFinite-op0,kernel_graph0,{10644-10647-10649},10649,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10610,10615,0x12c1f1c14600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9747,388_Default/clip_grad_norm-ClipGradNorm/Square-op20,Default/clip_grad_norm-ClipGradNorm/Square-op20,kernel_graph0,{10613},10613,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10612,10640,0x12c18000bc00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9749,389_Default/clip_grad_norm-ClipGradNorm/SumExt-op20,Default/clip_grad_norm-ClipGradNorm/SumExt-op20,kernel_graph0,{10616-10618},10618,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10613,10616,0x12c18000be00,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9749,389_Default/clip_grad_norm-ClipGradNorm/SumExt-op20,Default/clip_grad_norm-ClipGradNorm/SumExt-op20,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10618,10643,0x12c18000be00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9750,391_Default/clip_grad_norm-ClipGradNorm/AddN-op0,Default/clip_grad_norm-ClipGradNorm/AddN-op0,kernel_graph0,{10641},10641,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10641,13970,0x12c18000bc00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9752,392_Default/clip_grad_norm-ClipGradNorm/Sqrt-op0,Default/clip_grad_norm-ClipGradNorm/Sqrt-op0,kernel_graph0,{10651-10653},10653,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10644,10674,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9753,393_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Select-op1,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Select-op1,kernel_graph0,{10665-10670},10670,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10647,10791,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9754,394_Default/LogicalNot-op0,Default/LogicalNot-op0,kernel_graph0,{10667},10667,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10649,10677,0x12c180009200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9756,395_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/LogicalOr-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/LogicalOr-op0,kernel_graph0,{10675},10675,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10651,10655,0x12c180009a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9757,396_Default/clip_grad_norm-ClipGradNorm/GreaterEqual-op0,Default/clip_grad_norm-ClipGradNorm/GreaterEqual-op0,kernel_graph0,{10653},10653,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10653,10658,0x12c180009c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9757,397_Default/clip_grad_norm-ClipGradNorm/Select-op0,Default/clip_grad_norm-ClipGradNorm/Select-op0,kernel_graph0,{10656},10656,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10656,10661,0x12c180009a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9759,398_Default/learning_rate-CosineWithWarmUpLR/Add-op0,Default/learning_rate-CosineWithWarmUpLR/Add-op0,kernel_graph0,{10659},10659,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10659,10664,0x12c180009c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9760,399_Default/clip_grad_norm-ClipGradNorm/Div-op0,Default/clip_grad_norm-ClipGradNorm/Div-op0,kernel_graph0,{10662},10662,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10662,10743,0x12c180009a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9760,400_Default/clip_grad_norm-ClipGradNorm/Minimum-op0,Default/clip_grad_norm-ClipGradNorm/Minimum-op0,kernel_graph0,{10680-10683-10686-10689-10692-10695-10698-10701-10704-10707-10710-10713-10716-10719-10722-10725-10728-10731-10734-10737-10740},10740,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10665,10673,0x12c180009c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9763,401_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Mul-op1,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Mul-op1,kernel_graph0,{10670},10670,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10667,10672,0x12c180009e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9763,402_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/LogicalAnd-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/LogicalAnd-op0,kernel_graph0,{10670},10670,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10670,10749,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9764,403_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Select-op2,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Select-op2,kernel_graph0,{10747},10747,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10675,10745,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9764,404_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Select-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Select-op0,kernel_graph0,{10743},10743,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10678,10747,0x12c180009200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9765,405_Default/learning_rate-CosineWithWarmUpLR/Add-op1,Default/learning_rate-CosineWithWarmUpLR/Add-op1,kernel_graph0,{10745},10745,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10680,10950,0x12c1a6d35600,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9766,406_Default/clip_grad_norm-ClipGradNorm/Mul-op0,Default/clip_grad_norm-ClipGradNorm/Mul-op0,kernel_graph0,{10946-10948},10948,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10683,10940,0x12c209e10600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9766,407_Default/clip_grad_norm-ClipGradNorm/Mul-op1,Default/clip_grad_norm-ClipGradNorm/Mul-op1,kernel_graph0,{10936-10938},10938,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10686,10932,0x12c1f1c14000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9767,408_Default/clip_grad_norm-ClipGradNorm/Mul-op2,Default/clip_grad_norm-ClipGradNorm/Mul-op2,kernel_graph0,{10928-10930},10930,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10689,10924,0x12c1e9c0fa00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9768,409_Default/clip_grad_norm-ClipGradNorm/Mul-op3,Default/clip_grad_norm-ClipGradNorm/Mul-op3,kernel_graph0,{10920-10922},10922,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10692,10916,0x12c1f5c14200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9768,410_Default/clip_grad_norm-ClipGradNorm/Mul-op4,Default/clip_grad_norm-ClipGradNorm/Mul-op4,kernel_graph0,{10912-10914},10914,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10695,10908,0x12c1c6135800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9768,411_Default/clip_grad_norm-ClipGradNorm/Mul-op5,Default/clip_grad_norm-ClipGradNorm/Mul-op5,kernel_graph0,{10904-10906},10906,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10698,10900,0x12c1d0d35a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9768,412_Default/clip_grad_norm-ClipGradNorm/Mul-op6,Default/clip_grad_norm-ClipGradNorm/Mul-op6,kernel_graph0,{10896-10898},10898,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10701,10892,0x12c20de10800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9769,413_Default/clip_grad_norm-ClipGradNorm/Mul-op7,Default/clip_grad_norm-ClipGradNorm/Mul-op7,kernel_graph0,{10888-10890},10890,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10704,10884,0x12c23d611e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9769,414_Default/clip_grad_norm-ClipGradNorm/Mul-op8,Default/clip_grad_norm-ClipGradNorm/Mul-op8,kernel_graph0,{10880-10882},10882,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10707,10876,0x12c1db935c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9770,415_Default/clip_grad_norm-ClipGradNorm/Mul-op9,Default/clip_grad_norm-ClipGradNorm/Mul-op9,kernel_graph0,{10872-10874},10874,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10710,10868,0x12c1edc0fc00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9770,416_Default/clip_grad_norm-ClipGradNorm/Mul-op10,Default/clip_grad_norm-ClipGradNorm/Mul-op10,kernel_graph0,{10864-10866},10866,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10713,10860,0x12c1df935e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9773,417_Default/clip_grad_norm-ClipGradNorm/Mul-op11,Default/clip_grad_norm-ClipGradNorm/Mul-op11,kernel_graph0,{10856-10858},10858,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10716,10852,0x12c224a11200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9774,418_Default/clip_grad_norm-ClipGradNorm/Mul-op12,Default/clip_grad_norm-ClipGradNorm/Mul-op12,kernel_graph0,{10848-10850},10850,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10719,10844,0x12c22f611400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9775,419_Default/clip_grad_norm-ClipGradNorm/Mul-op13,Default/clip_grad_norm-ClipGradNorm/Mul-op13,kernel_graph0,{10840-10842},10842,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10722,10836,0x12c218a10a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9775,420_Default/clip_grad_norm-ClipGradNorm/Mul-op14,Default/clip_grad_norm-ClipGradNorm/Mul-op14,kernel_graph0,{10832-10834},10834,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10725,10828,0x12c180018000,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9775,421_Default/clip_grad_norm-ClipGradNorm/Mul-op15,Default/clip_grad_norm-ClipGradNorm/Mul-op15,kernel_graph0,{10824-10826},10826,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10728,10820,0x12c223610c00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9775,422_Default/clip_grad_norm-ClipGradNorm/Mul-op16,Default/clip_grad_norm-ClipGradNorm/Mul-op16,kernel_graph0,{10816-10818},10818,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10731,10812,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9776,423_Default/clip_grad_norm-ClipGradNorm/Mul-op17,Default/clip_grad_norm-ClipGradNorm/Mul-op17,kernel_graph0,{10808-10810},10810,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10734,10804,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9778,424_Default/clip_grad_norm-ClipGradNorm/Mul-op18,Default/clip_grad_norm-ClipGradNorm/Mul-op18,kernel_graph0,{10800-10802},10802,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10737,10796,0x12c180013e00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9779,425_Default/clip_grad_norm-ClipGradNorm/Mul-op19,Default/clip_grad_norm-ClipGradNorm/Mul-op19,kernel_graph0,{10792-10794},10794,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10740,10958,0x12c18000be00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9780,426_Default/clip_grad_norm-ClipGradNorm/Mul-op20,Default/clip_grad_norm-ClipGradNorm/Mul-op20,kernel_graph0,{10954-10956},10956,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10750,10759,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9784,430_Default/learning_rate-CosineWithWarmUpLR/Cast-op0,Default/learning_rate-CosineWithWarmUpLR/Cast-op0,kernel_graph0,{10752-10755-10757},10757,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10752,10754,0x12c180009200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,9786,431_Default/learning_rate-CosineWithWarmUpLR/GreaterEqual-op0,Default/learning_rate-CosineWithWarmUpLR/GreaterEqual-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10755,10760,0x12c180009200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10754,435_Default/learning_rate-CosineWithWarmUpLR/Greater-op0,Default/learning_rate-CosineWithWarmUpLR/Greater-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10757,10763,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10755,436_Default/TensorMove-op0,Default/TensorMove-op0,kernel_graph0,{10761},10761,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10761,10766,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10760,440_Default/learning_rate-CosineWithWarmUpLR/Sub-op1,Default/learning_rate-CosineWithWarmUpLR/Sub-op1,kernel_graph0,{10764},10764,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10764,10769,0x12c180009200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10761,441_Default/learning_rate-CosineWithWarmUpLR/Div-op1,Default/learning_rate-CosineWithWarmUpLR/Div-op1,kernel_graph0,{10767},10767,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10767,10772,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10762,442_Default/learning_rate-CosineWithWarmUpLR/Mul-op1,Default/learning_rate-CosineWithWarmUpLR/Mul-op1,kernel_graph0,{10770},10770,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10770,10775,0x12c180009200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10762,443_Default/learning_rate-CosineWithWarmUpLR/Cos-op0,Default/learning_rate-CosineWithWarmUpLR/Cos-op0,kernel_graph0,{10773},10773,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10773,10778,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10764,444_Default/learning_rate-CosineWithWarmUpLR/Add-op3,Default/learning_rate-CosineWithWarmUpLR/Add-op3,kernel_graph0,{10776},10776,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10776,10781,0x12c180009200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10765,445_Default/learning_rate-CosineWithWarmUpLR/Mul-op2,Default/learning_rate-CosineWithWarmUpLR/Mul-op2,kernel_graph0,{10779},10779,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10779,10784,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10766,446_Default/learning_rate-CosineWithWarmUpLR/Maximum-op0,Default/learning_rate-CosineWithWarmUpLR/Maximum-op0,kernel_graph0,{10782},10782,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10782,10787,0x12c180009200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10767,447_Default/learning_rate-CosineWithWarmUpLR/Mul-op3,Default/learning_rate-CosineWithWarmUpLR/Mul-op3,kernel_graph0,{10785},10785,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10785,10790,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10768,448_Default/learning_rate-CosineWithWarmUpLR/Add-op4,Default/learning_rate-CosineWithWarmUpLR/Add-op4,kernel_graph0,{10788},10788,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10788,13971,0x12c180009200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10771,451_Default/Reshape-op0,Default/Reshape-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10792,11621,0x12c1f1c0fe00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10791,453_Default/optimizer-FP32StateAdamWeightDecay/Mul-op96,Default/optimizer-FP32StateAdamWeightDecay/Mul-op96,kernel_graph0,{11618},11618,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10794,10799,0x12c223614e00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10792,454_Default/optimizer-FP32StateAdamWeightDecay/Square-op19,Default/optimizer-FP32StateAdamWeightDecay/Square-op19,kernel_graph0,{10797},10797,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10797,11617,0x12c180010000,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10793,455_Default/optimizer-FP32StateAdamWeightDecay/Mul-op98,Default/optimizer-FP32StateAdamWeightDecay/Mul-op98,kernel_graph0,{11614},11614,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10800,11585,0x12c223614e00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10794,456_Default/optimizer-FP32StateAdamWeightDecay/Mul-op91,Default/optimizer-FP32StateAdamWeightDecay/Mul-op91,kernel_graph0,{11582},11582,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10802,10807,0x12c223619000,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10795,457_Default/optimizer-FP32StateAdamWeightDecay/Square-op18,Default/optimizer-FP32StateAdamWeightDecay/Square-op18,kernel_graph0,{10805},10805,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10805,11589,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10795,458_Default/optimizer-FP32StateAdamWeightDecay/Mul-op93,Default/optimizer-FP32StateAdamWeightDecay/Mul-op93,kernel_graph0,{11586},11586,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10808,11553,0x12c223619000,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10797,459_Default/optimizer-FP32StateAdamWeightDecay/Mul-op86,Default/optimizer-FP32StateAdamWeightDecay/Mul-op86,kernel_graph0,{11550},11550,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10810,10815,0x12c22361d200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10798,460_Default/optimizer-FP32StateAdamWeightDecay/Square-op17,Default/optimizer-FP32StateAdamWeightDecay/Square-op17,kernel_graph0,{10813},10813,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10813,11557,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10798,461_Default/optimizer-FP32StateAdamWeightDecay/Mul-op88,Default/optimizer-FP32StateAdamWeightDecay/Mul-op88,kernel_graph0,{11554},11554,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10816,11521,0x12c22361d200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10800,462_Default/optimizer-FP32StateAdamWeightDecay/Mul-op81,Default/optimizer-FP32StateAdamWeightDecay/Mul-op81,kernel_graph0,{11518},11518,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10818,10823,0x12c223621400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10801,463_Default/optimizer-FP32StateAdamWeightDecay/Square-op16,Default/optimizer-FP32StateAdamWeightDecay/Square-op16,kernel_graph0,{10821},10821,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10821,11525,0x12c223610c00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10801,464_Default/optimizer-FP32StateAdamWeightDecay/Mul-op83,Default/optimizer-FP32StateAdamWeightDecay/Mul-op83,kernel_graph0,{11522},11522,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10824,11489,0x12c241612000,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10802,465_Default/optimizer-FP32StateAdamWeightDecay/Mul-op76,Default/optimizer-FP32StateAdamWeightDecay/Mul-op76,kernel_graph0,{11486},11486,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10826,10831,0x12c260a12200,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10803,466_Default/optimizer-FP32StateAdamWeightDecay/Square-op15,Default/optimizer-FP32StateAdamWeightDecay/Square-op15,kernel_graph0,{10829},10829,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10829,11493,0x12c180014200,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10803,467_Default/optimizer-FP32StateAdamWeightDecay/Mul-op78,Default/optimizer-FP32StateAdamWeightDecay/Mul-op78,kernel_graph0,{11490},11490,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10832,11457,0x12c1f9c14400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10805,468_Default/optimizer-FP32StateAdamWeightDecay/Mul-op71,Default/optimizer-FP32StateAdamWeightDecay/Mul-op71,kernel_graph0,{11454},11454,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10834,10839,0x12c260a12200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10806,469_Default/optimizer-FP32StateAdamWeightDecay/Square-op14,Default/optimizer-FP32StateAdamWeightDecay/Square-op14,kernel_graph0,{10837},10837,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10837,11461,0x12c218a10a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10806,470_Default/optimizer-FP32StateAdamWeightDecay/Mul-op73,Default/optimizer-FP32StateAdamWeightDecay/Mul-op73,kernel_graph0,{11458},11458,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10840,11425,0x12c260a12200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10808,471_Default/optimizer-FP32StateAdamWeightDecay/Mul-op66,Default/optimizer-FP32StateAdamWeightDecay/Mul-op66,kernel_graph0,{11422},11422,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10842,10847,0x12c26b612400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10809,472_Default/optimizer-FP32StateAdamWeightDecay/Square-op13,Default/optimizer-FP32StateAdamWeightDecay/Square-op13,kernel_graph0,{10845},10845,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10845,11429,0x12c22f611400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10809,473_Default/optimizer-FP32StateAdamWeightDecay/Mul-op68,Default/optimizer-FP32StateAdamWeightDecay/Mul-op68,kernel_graph0,{11426},11426,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10848,11393,0x12c26b612400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10810,474_Default/optimizer-FP32StateAdamWeightDecay/Mul-op61,Default/optimizer-FP32StateAdamWeightDecay/Mul-op61,kernel_graph0,{11390},11390,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10850,10855,0x12c276212600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10811,475_Default/optimizer-FP32StateAdamWeightDecay/Square-op12,Default/optimizer-FP32StateAdamWeightDecay/Square-op12,kernel_graph0,{10853},10853,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10853,11397,0x12c223621400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10811,476_Default/optimizer-FP32StateAdamWeightDecay/Mul-op63,Default/optimizer-FP32StateAdamWeightDecay/Mul-op63,kernel_graph0,{11394},11394,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10856,11361,0x12c204814600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10813,477_Default/optimizer-FP32StateAdamWeightDecay/Mul-op56,Default/optimizer-FP32StateAdamWeightDecay/Mul-op56,kernel_graph0,{11358},11358,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10858,10863,0x12c1e3936000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10814,478_Default/optimizer-FP32StateAdamWeightDecay/Square-op11,Default/optimizer-FP32StateAdamWeightDecay/Square-op11,kernel_graph0,{10861},10861,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10861,11365,0x12c1df935e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10814,479_Default/optimizer-FP32StateAdamWeightDecay/Mul-op58,Default/optimizer-FP32StateAdamWeightDecay/Mul-op58,kernel_graph0,{11362},11362,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10864,11329,0x12c1e3936000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10816,480_Default/optimizer-FP32StateAdamWeightDecay/Mul-op51,Default/optimizer-FP32StateAdamWeightDecay/Mul-op51,kernel_graph0,{11326},11326,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10866,10871,0x12c19f414400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10817,481_Default/optimizer-FP32StateAdamWeightDecay/Square-op10,Default/optimizer-FP32StateAdamWeightDecay/Square-op10,kernel_graph0,{10869},10869,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10869,11333,0x12c1edc0fc00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10817,482_Default/optimizer-FP32StateAdamWeightDecay/Mul-op53,Default/optimizer-FP32StateAdamWeightDecay/Mul-op53,kernel_graph0,{11330},11330,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10872,11297,0x12c19f414400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10818,483_Default/optimizer-FP32StateAdamWeightDecay/Mul-op46,Default/optimizer-FP32StateAdamWeightDecay/Mul-op46,kernel_graph0,{11294},11294,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10874,10879,0x12c276212600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10819,484_Default/optimizer-FP32StateAdamWeightDecay/Square-op9,Default/optimizer-FP32StateAdamWeightDecay/Square-op9,kernel_graph0,{10877},10877,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10877,11301,0x12c1db935c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10819,485_Default/optimizer-FP32StateAdamWeightDecay/Mul-op48,Default/optimizer-FP32StateAdamWeightDecay/Mul-op48,kernel_graph0,{11298},11298,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10880,11265,0x12c276212600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10821,486_Default/optimizer-FP32StateAdamWeightDecay/Mul-op41,Default/optimizer-FP32StateAdamWeightDecay/Mul-op41,kernel_graph0,{11262},11262,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10882,10887,0x12c27a212800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10822,487_Default/optimizer-FP32StateAdamWeightDecay/Square-op8,Default/optimizer-FP32StateAdamWeightDecay/Square-op8,kernel_graph0,{10885},10885,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10885,11269,0x12c23a211600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10822,488_Default/optimizer-FP32StateAdamWeightDecay/Mul-op43,Default/optimizer-FP32StateAdamWeightDecay/Mul-op43,kernel_graph0,{11266},11266,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10888,11233,0x12c27a212800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10824,489_Default/optimizer-FP32StateAdamWeightDecay/Mul-op36,Default/optimizer-FP32StateAdamWeightDecay/Mul-op36,kernel_graph0,{11230},11230,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10890,10895,0x12c284e12a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10825,490_Default/optimizer-FP32StateAdamWeightDecay/Square-op7,Default/optimizer-FP32StateAdamWeightDecay/Square-op7,kernel_graph0,{10893},10893,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10893,11237,0x12c20de10800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10825,491_Default/optimizer-FP32StateAdamWeightDecay/Mul-op38,Default/optimizer-FP32StateAdamWeightDecay/Mul-op38,kernel_graph0,{11234},11234,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10896,11201,0x12c284e12a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10826,492_Default/optimizer-FP32StateAdamWeightDecay/Mul-op31,Default/optimizer-FP32StateAdamWeightDecay/Mul-op31,kernel_graph0,{11198},11198,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10898,10903,0x12c28fa12c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10827,493_Default/optimizer-FP32StateAdamWeightDecay/Square-op6,Default/optimizer-FP32StateAdamWeightDecay/Square-op6,kernel_graph0,{10901},10901,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10901,11205,0x12c1d0d35a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10828,494_Default/optimizer-FP32StateAdamWeightDecay/Mul-op33,Default/optimizer-FP32StateAdamWeightDecay/Mul-op33,kernel_graph0,{11202},11202,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10904,11169,0x12c28fa12c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10829,495_Default/optimizer-FP32StateAdamWeightDecay/Mul-op26,Default/optimizer-FP32StateAdamWeightDecay/Mul-op26,kernel_graph0,{11166},11166,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10906,10911,0x12c29a612e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10830,496_Default/optimizer-FP32StateAdamWeightDecay/Square-op5,Default/optimizer-FP32StateAdamWeightDecay/Square-op5,kernel_graph0,{10909},10909,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10909,11173,0x12c1c6135800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10831,497_Default/optimizer-FP32StateAdamWeightDecay/Mul-op28,Default/optimizer-FP32StateAdamWeightDecay/Mul-op28,kernel_graph0,{11170},11170,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10912,11137,0x12c29a612e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10832,498_Default/optimizer-FP32StateAdamWeightDecay/Mul-op21,Default/optimizer-FP32StateAdamWeightDecay/Mul-op21,kernel_graph0,{11134},11134,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10914,10919,0x12c29e613000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10833,499_Default/optimizer-FP32StateAdamWeightDecay/Square-op4,Default/optimizer-FP32StateAdamWeightDecay/Square-op4,kernel_graph0,{10917},10917,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10917,11141,0x12c1f5c14200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10834,500_Default/optimizer-FP32StateAdamWeightDecay/Mul-op23,Default/optimizer-FP32StateAdamWeightDecay/Mul-op23,kernel_graph0,{11138},11138,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10920,11105,0x12c29e613000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10835,501_Default/optimizer-FP32StateAdamWeightDecay/Mul-op16,Default/optimizer-FP32StateAdamWeightDecay/Mul-op16,kernel_graph0,{11102},11102,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10922,10927,0x12c2a2613200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10835,502_Default/optimizer-FP32StateAdamWeightDecay/Square-op3,Default/optimizer-FP32StateAdamWeightDecay/Square-op3,kernel_graph0,{10925},10925,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10925,11109,0x12c1e7936200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10837,503_Default/optimizer-FP32StateAdamWeightDecay/Mul-op18,Default/optimizer-FP32StateAdamWeightDecay/Mul-op18,kernel_graph0,{11106},11106,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10928,11073,0x12c2a2613200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10838,504_Default/optimizer-FP32StateAdamWeightDecay/Mul-op11,Default/optimizer-FP32StateAdamWeightDecay/Mul-op11,kernel_graph0,{11070},11070,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10930,10935,0x12c2a6613400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10838,505_Default/optimizer-FP32StateAdamWeightDecay/Square-op2,Default/optimizer-FP32StateAdamWeightDecay/Square-op2,kernel_graph0,{10933},10933,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10933,11077,0x12c1f1c14000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10840,506_Default/optimizer-FP32StateAdamWeightDecay/Mul-op13,Default/optimizer-FP32StateAdamWeightDecay/Mul-op13,kernel_graph0,{11074},11074,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10936,11041,0x12c2a6613400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10841,507_Default/optimizer-FP32StateAdamWeightDecay/Mul-op6,Default/optimizer-FP32StateAdamWeightDecay/Mul-op6,kernel_graph0,{11038},11038,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10938,10943,0x12c2aa613600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10841,508_Default/optimizer-FP32StateAdamWeightDecay/Square-op1,Default/optimizer-FP32StateAdamWeightDecay/Square-op1,kernel_graph0,{10941},10941,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10941,11045,0x12c208814800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10842,509_Default/optimizer-FP32StateAdamWeightDecay/Mul-op8,Default/optimizer-FP32StateAdamWeightDecay/Mul-op8,kernel_graph0,{11042},11042,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10944,10969,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10843,510_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Cast-op0,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Cast-op0,kernel_graph0,{10962-10965-10967},10967,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10946,11009,0x12c2aa613600,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10844,511_Default/optimizer-FP32StateAdamWeightDecay/Mul-op1,Default/optimizer-FP32StateAdamWeightDecay/Mul-op1,kernel_graph0,{11006},11006,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10948,10953,0x12c2c9a13800,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10845,512_Default/optimizer-FP32StateAdamWeightDecay/Square-op0,Default/optimizer-FP32StateAdamWeightDecay/Square-op0,kernel_graph0,{10951},10951,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10951,11013,0x12c1a3414600,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10846,513_Default/optimizer-FP32StateAdamWeightDecay/Mul-op3,Default/optimizer-FP32StateAdamWeightDecay/Mul-op3,kernel_graph0,{11010},11010,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10954,11653,0x12c22e221600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10847,514_Default/optimizer-FP32StateAdamWeightDecay/Mul-op101,Default/optimizer-FP32StateAdamWeightDecay/Mul-op101,kernel_graph0,{11650},11650,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10956,10961,0x12c22e225800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10848,515_Default/optimizer-FP32StateAdamWeightDecay/Square-op20,Default/optimizer-FP32StateAdamWeightDecay/Square-op20,kernel_graph0,{10959},10959,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10959,11649,0x12c18000be00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10849,516_Default/optimizer-FP32StateAdamWeightDecay/Mul-op103,Default/optimizer-FP32StateAdamWeightDecay/Mul-op103,kernel_graph0,{11646},11646,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10962,10964,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10850,518_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/GreaterEqual-op0,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/GreaterEqual-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10965,10970,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10964,522_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Greater-op0,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Greater-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10967,10973,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10965,523_Default/TensorMove-op1,Default/TensorMove-op1,kernel_graph0,{10971},10971,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10971,10976,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10970,527_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Sub-op1,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Sub-op1,kernel_graph0,{10974},10974,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10974,10979,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10971,528_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Div-op1,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Div-op1,kernel_graph0,{10977},10977,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10977,10982,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10972,529_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Mul-op1,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Mul-op1,kernel_graph0,{10980},10980,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10980,10985,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10972,530_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Cos-op0,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Cos-op0,kernel_graph0,{10983},10983,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10983,10988,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10974,531_Default/learning_rate-CosineWithWarmUpLR/Add-op6,Default/learning_rate-CosineWithWarmUpLR/Add-op6,kernel_graph0,{10986},10986,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10986,10991,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10975,532_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Mul-op2,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Mul-op2,kernel_graph0,{10989},10989,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10989,10994,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10975,533_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Maximum-op0,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Maximum-op0,kernel_graph0,{10992},10992,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10992,10997,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10977,534_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Mul-op3,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Mul-op3,kernel_graph0,{10995},10995,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10995,11000,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10978,535_Default/learning_rate-CosineWithWarmUpLR/Add-op7,Default/learning_rate-CosineWithWarmUpLR/Add-op7,kernel_graph0,{10998},10998,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 10998,11664,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10981,538_Default/optimizer-FP32StateAdamWeightDecay/Reshape-op0,Default/optimizer-FP32StateAdamWeightDecay/Reshape-op0,kernel_graph0,{11022-11054-11086-11118-11150-11182-11214-11246-11278-11310-11342-11374-11406-11438-11470-11502-11534-11566-11598-11630-11662},11662,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11002,11008,0x12c2c9a13800,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10983,540_Default/optimizer-FP32StateAdamWeightDecay/Mul-op0,Default/optimizer-FP32StateAdamWeightDecay/Mul-op0,kernel_graph0,{11006},11006,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11004,11012,0x12c2e8e13a00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10984,541_Default/optimizer-FP32StateAdamWeightDecay/Mul-op2,Default/optimizer-FP32StateAdamWeightDecay/Mul-op2,kernel_graph0,{11010},11010,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11006,11031,0x12c308213c00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10984,542_Default/learning_rate-CosineWithWarmUpLR/Add-op8,Default/learning_rate-CosineWithWarmUpLR/Add-op8,kernel_graph0,{11019-11029},11029,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11010,11029,0x12c2aa613600,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10986,543_Default/learning_rate-CosineWithWarmUpLR/Add-op9,Default/learning_rate-CosineWithWarmUpLR/Add-op9,kernel_graph0,{11014-11027},11027,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11014,11018,0x12c1a3414600,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10987,544_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op0,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op0,kernel_graph0,{11016},11016,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11016,11021,0x12c2c9a13800,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10987,545_Default/learning_rate-CosineWithWarmUpLR/Add-op10,Default/learning_rate-CosineWithWarmUpLR/Add-op10,kernel_graph0,{11019},11019,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11019,11024,0x12c2e8e13a00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10989,546_Default/optimizer-FP32StateAdamWeightDecay/Div-op0,Default/optimizer-FP32StateAdamWeightDecay/Div-op0,kernel_graph0,{11022},11022,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11022,11027,0x12c2c9a13800,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10990,547_Default/optimizer-FP32StateAdamWeightDecay/Mul-op4,Default/optimizer-FP32StateAdamWeightDecay/Mul-op4,kernel_graph0,{11025},11025,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11025,11033,0x12c2e8e13a00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10990,548_Default/optimizer-FP32StateAdamWeightDecay/Sub-op0,Default/optimizer-FP32StateAdamWeightDecay/Sub-op0,kernel_graph0,{11031},11031,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11034,11040,0x12c1a3414600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10995,552_Default/optimizer-FP32StateAdamWeightDecay/Mul-op5,Default/optimizer-FP32StateAdamWeightDecay/Mul-op5,kernel_graph0,{11038},11038,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11036,11044,0x12c1a7414800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10996,553_Default/optimizer-FP32StateAdamWeightDecay/Mul-op7,Default/optimizer-FP32StateAdamWeightDecay/Mul-op7,kernel_graph0,{11042},11042,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11038,11063,0x12c1ab414a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10996,554_Default/learning_rate-CosineWithWarmUpLR/Add-op11,Default/learning_rate-CosineWithWarmUpLR/Add-op11,kernel_graph0,{11051-11061},11061,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11042,11061,0x12c1a3414600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10998,555_Default/learning_rate-CosineWithWarmUpLR/Add-op12,Default/learning_rate-CosineWithWarmUpLR/Add-op12,kernel_graph0,{11046-11059},11059,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11046,11050,0x12c1a7414800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,10999,556_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op1,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op1,kernel_graph0,{11048},11048,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11048,11053,0x12c208814800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11000,557_Default/learning_rate-CosineWithWarmUpLR/Add-op13,Default/learning_rate-CosineWithWarmUpLR/Add-op13,kernel_graph0,{11051},11051,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11051,11056,0x12c1a7414800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11000,558_Default/optimizer-FP32StateAdamWeightDecay/Div-op1,Default/optimizer-FP32StateAdamWeightDecay/Div-op1,kernel_graph0,{11054},11054,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11054,11059,0x12c208814800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11001,559_Default/optimizer-FP32StateAdamWeightDecay/Mul-op9,Default/optimizer-FP32StateAdamWeightDecay/Mul-op9,kernel_graph0,{11057},11057,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11057,11065,0x12c1a7414800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11001,560_Default/optimizer-FP32StateAdamWeightDecay/Sub-op1,Default/optimizer-FP32StateAdamWeightDecay/Sub-op1,kernel_graph0,{11063},11063,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11066,11072,0x12c208814800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11003,564_Default/optimizer-FP32StateAdamWeightDecay/Mul-op10,Default/optimizer-FP32StateAdamWeightDecay/Mul-op10,kernel_graph0,{11070},11070,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11068,11076,0x12c1a3414600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11003,565_Default/optimizer-FP32StateAdamWeightDecay/Mul-op12,Default/optimizer-FP32StateAdamWeightDecay/Mul-op12,kernel_graph0,{11074},11074,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11070,11095,0x12c1a7414800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11004,566_Default/learning_rate-CosineWithWarmUpLR/Add-op14,Default/learning_rate-CosineWithWarmUpLR/Add-op14,kernel_graph0,{11083-11093},11093,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11074,11093,0x12c208814800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11005,567_Default/learning_rate-CosineWithWarmUpLR/Add-op15,Default/learning_rate-CosineWithWarmUpLR/Add-op15,kernel_graph0,{11078-11091},11091,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11078,11082,0x12c1a3414600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11005,568_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op2,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op2,kernel_graph0,{11080},11080,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11080,11085,0x12c1f1c14000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11006,569_Default/learning_rate-CosineWithWarmUpLR/Add-op16,Default/learning_rate-CosineWithWarmUpLR/Add-op16,kernel_graph0,{11083},11083,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11083,11088,0x12c1a3414600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11007,570_Default/optimizer-FP32StateAdamWeightDecay/Div-op2,Default/optimizer-FP32StateAdamWeightDecay/Div-op2,kernel_graph0,{11086},11086,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11086,11091,0x12c1f1c14000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11007,571_Default/optimizer-FP32StateAdamWeightDecay/Mul-op14,Default/optimizer-FP32StateAdamWeightDecay/Mul-op14,kernel_graph0,{11089},11089,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11089,11097,0x12c1a3414600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11010,572_Default/optimizer-FP32StateAdamWeightDecay/Sub-op2,Default/optimizer-FP32StateAdamWeightDecay/Sub-op2,kernel_graph0,{11095},11095,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11098,11104,0x12c1f1c14000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11015,576_Default/optimizer-FP32StateAdamWeightDecay/Mul-op15,Default/optimizer-FP32StateAdamWeightDecay/Mul-op15,kernel_graph0,{11102},11102,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11100,11108,0x12c208814800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11015,577_Default/optimizer-FP32StateAdamWeightDecay/Mul-op17,Default/optimizer-FP32StateAdamWeightDecay/Mul-op17,kernel_graph0,{11106},11106,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11102,11127,0x12c1a3414600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11016,578_Default/learning_rate-CosineWithWarmUpLR/Add-op17,Default/learning_rate-CosineWithWarmUpLR/Add-op17,kernel_graph0,{11115-11125},11125,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11106,11125,0x12c1f1c14000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11017,579_Default/learning_rate-CosineWithWarmUpLR/Add-op18,Default/learning_rate-CosineWithWarmUpLR/Add-op18,kernel_graph0,{11110-11123},11123,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11110,11114,0x12c208814800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11018,580_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op3,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op3,kernel_graph0,{11112},11112,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11112,11117,0x12c1e7936200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11019,581_Default/learning_rate-CosineWithWarmUpLR/Add-op19,Default/learning_rate-CosineWithWarmUpLR/Add-op19,kernel_graph0,{11115},11115,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11115,11120,0x12c208814800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11020,582_Default/optimizer-FP32StateAdamWeightDecay/Div-op3,Default/optimizer-FP32StateAdamWeightDecay/Div-op3,kernel_graph0,{11118},11118,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11118,11123,0x12c1e7936200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11020,583_Default/optimizer-FP32StateAdamWeightDecay/Mul-op19,Default/optimizer-FP32StateAdamWeightDecay/Mul-op19,kernel_graph0,{11121},11121,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11121,11129,0x12c208814800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11022,584_Default/optimizer-FP32StateAdamWeightDecay/Sub-op3,Default/optimizer-FP32StateAdamWeightDecay/Sub-op3,kernel_graph0,{11127},11127,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11130,11136,0x12c1f1c14000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11026,588_Default/optimizer-FP32StateAdamWeightDecay/Mul-op20,Default/optimizer-FP32StateAdamWeightDecay/Mul-op20,kernel_graph0,{11134},11134,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11132,11140,0x12c208814800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11026,589_Default/optimizer-FP32StateAdamWeightDecay/Mul-op22,Default/optimizer-FP32StateAdamWeightDecay/Mul-op22,kernel_graph0,{11138},11138,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11134,11159,0x12c1e7936200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11027,590_Default/learning_rate-CosineWithWarmUpLR/Add-op20,Default/learning_rate-CosineWithWarmUpLR/Add-op20,kernel_graph0,{11147-11157},11157,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11138,11157,0x12c1f1c14000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11028,591_Default/learning_rate-CosineWithWarmUpLR/Add-op21,Default/learning_rate-CosineWithWarmUpLR/Add-op21,kernel_graph0,{11142-11155},11155,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11142,11146,0x12c1f5c14200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11028,592_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op4,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op4,kernel_graph0,{11144},11144,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11144,11149,0x12c208814800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11028,593_Default/learning_rate-CosineWithWarmUpLR/Add-op22,Default/learning_rate-CosineWithWarmUpLR/Add-op22,kernel_graph0,{11147},11147,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11147,11152,0x12c1f5c14200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11028,594_Default/optimizer-FP32StateAdamWeightDecay/Div-op4,Default/optimizer-FP32StateAdamWeightDecay/Div-op4,kernel_graph0,{11150},11150,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11150,11155,0x12c208814800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11029,595_Default/optimizer-FP32StateAdamWeightDecay/Mul-op24,Default/optimizer-FP32StateAdamWeightDecay/Mul-op24,kernel_graph0,{11153},11153,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11153,11161,0x12c1f5c14200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11030,596_Default/optimizer-FP32StateAdamWeightDecay/Sub-op4,Default/optimizer-FP32StateAdamWeightDecay/Sub-op4,kernel_graph0,{11159},11159,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11162,11168,0x12c1a3414600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11032,600_Default/optimizer-FP32StateAdamWeightDecay/Mul-op25,Default/optimizer-FP32StateAdamWeightDecay/Mul-op25,kernel_graph0,{11166},11166,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11164,11172,0x12c1ae014800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11032,601_Default/optimizer-FP32StateAdamWeightDecay/Mul-op27,Default/optimizer-FP32StateAdamWeightDecay/Mul-op27,kernel_graph0,{11170},11170,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11166,11191,0x12c1b8c14a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11032,602_Default/learning_rate-CosineWithWarmUpLR/Add-op23,Default/learning_rate-CosineWithWarmUpLR/Add-op23,kernel_graph0,{11179-11189},11189,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11170,11189,0x12c1a3414600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11034,603_Default/learning_rate-CosineWithWarmUpLR/Add-op24,Default/learning_rate-CosineWithWarmUpLR/Add-op24,kernel_graph0,{11174-11187},11187,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11174,11178,0x12c1ae014800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11034,604_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op5,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op5,kernel_graph0,{11176},11176,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11176,11181,0x12c1c3814c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11035,605_Default/learning_rate-CosineWithWarmUpLR/Add-op25,Default/learning_rate-CosineWithWarmUpLR/Add-op25,kernel_graph0,{11179},11179,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11179,11184,0x12c1ae014800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11036,606_Default/optimizer-FP32StateAdamWeightDecay/Div-op5,Default/optimizer-FP32StateAdamWeightDecay/Div-op5,kernel_graph0,{11182},11182,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11182,11187,0x12c1c3814c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11036,607_Default/optimizer-FP32StateAdamWeightDecay/Mul-op29,Default/optimizer-FP32StateAdamWeightDecay/Mul-op29,kernel_graph0,{11185},11185,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11185,11193,0x12c1ae014800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11037,608_Default/optimizer-FP32StateAdamWeightDecay/Sub-op5,Default/optimizer-FP32StateAdamWeightDecay/Sub-op5,kernel_graph0,{11191},11191,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11194,11200,0x12c1a3414600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11041,612_Default/optimizer-FP32StateAdamWeightDecay/Mul-op30,Default/optimizer-FP32StateAdamWeightDecay/Mul-op30,kernel_graph0,{11198},11198,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11196,11204,0x12c1ae014800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11042,613_Default/optimizer-FP32StateAdamWeightDecay/Mul-op32,Default/optimizer-FP32StateAdamWeightDecay/Mul-op32,kernel_graph0,{11202},11202,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11198,11223,0x12c1b8c14a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11043,614_Default/learning_rate-CosineWithWarmUpLR/Add-op26,Default/learning_rate-CosineWithWarmUpLR/Add-op26,kernel_graph0,{11211-11221},11221,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11202,11221,0x12c1a3414600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11043,615_Default/learning_rate-CosineWithWarmUpLR/Add-op27,Default/learning_rate-CosineWithWarmUpLR/Add-op27,kernel_graph0,{11206-11219},11219,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11206,11210,0x12c1ae014800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11046,616_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op6,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op6,kernel_graph0,{11208},11208,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11208,11213,0x12c1c3814c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11047,617_Default/learning_rate-CosineWithWarmUpLR/Add-op28,Default/learning_rate-CosineWithWarmUpLR/Add-op28,kernel_graph0,{11211},11211,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11211,11216,0x12c1ae014800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11047,618_Default/optimizer-FP32StateAdamWeightDecay/Div-op6,Default/optimizer-FP32StateAdamWeightDecay/Div-op6,kernel_graph0,{11214},11214,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11214,11219,0x12c1c3814c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11048,619_Default/optimizer-FP32StateAdamWeightDecay/Mul-op34,Default/optimizer-FP32StateAdamWeightDecay/Mul-op34,kernel_graph0,{11217},11217,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11217,11225,0x12c1ae014800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11049,620_Default/optimizer-FP32StateAdamWeightDecay/Sub-op6,Default/optimizer-FP32StateAdamWeightDecay/Sub-op6,kernel_graph0,{11223},11223,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11226,11232,0x12c1a3414600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11054,624_Default/optimizer-FP32StateAdamWeightDecay/Mul-op35,Default/optimizer-FP32StateAdamWeightDecay/Mul-op35,kernel_graph0,{11230},11230,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11228,11236,0x12c1ae014800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11054,625_Default/optimizer-FP32StateAdamWeightDecay/Mul-op37,Default/optimizer-FP32StateAdamWeightDecay/Mul-op37,kernel_graph0,{11234},11234,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11230,11255,0x12c1b8c14a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11055,626_Default/learning_rate-CosineWithWarmUpLR/Add-op29,Default/learning_rate-CosineWithWarmUpLR/Add-op29,kernel_graph0,{11243-11253},11253,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11234,11253,0x12c1a3414600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11056,627_Default/learning_rate-CosineWithWarmUpLR/Add-op30,Default/learning_rate-CosineWithWarmUpLR/Add-op30,kernel_graph0,{11238-11251},11251,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11238,11242,0x12c1ae014800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11057,628_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op7,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op7,kernel_graph0,{11240},11240,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11240,11245,0x12c208814800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11058,629_Default/learning_rate-CosineWithWarmUpLR/Add-op31,Default/learning_rate-CosineWithWarmUpLR/Add-op31,kernel_graph0,{11243},11243,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11243,11248,0x12c1ae014800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11058,630_Default/optimizer-FP32StateAdamWeightDecay/Div-op7,Default/optimizer-FP32StateAdamWeightDecay/Div-op7,kernel_graph0,{11246},11246,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11246,11251,0x12c208814800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11059,631_Default/optimizer-FP32StateAdamWeightDecay/Mul-op39,Default/optimizer-FP32StateAdamWeightDecay/Mul-op39,kernel_graph0,{11249},11249,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11249,11257,0x12c1ae014800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11060,632_Default/optimizer-FP32StateAdamWeightDecay/Sub-op7,Default/optimizer-FP32StateAdamWeightDecay/Sub-op7,kernel_graph0,{11255},11255,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11258,11264,0x12c1e7936200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11062,636_Default/optimizer-FP32StateAdamWeightDecay/Mul-op40,Default/optimizer-FP32StateAdamWeightDecay/Mul-op40,kernel_graph0,{11262},11262,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11260,11268,0x12c1f1c14000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11062,637_Default/optimizer-FP32StateAdamWeightDecay/Mul-op42,Default/optimizer-FP32StateAdamWeightDecay/Mul-op42,kernel_graph0,{11266},11266,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11262,11287,0x12c1f5c14200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11062,638_Default/learning_rate-CosineWithWarmUpLR/Add-op32,Default/learning_rate-CosineWithWarmUpLR/Add-op32,kernel_graph0,{11275-11285},11285,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11266,11285,0x12c1e7936200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11063,639_Default/learning_rate-CosineWithWarmUpLR/Add-op33,Default/learning_rate-CosineWithWarmUpLR/Add-op33,kernel_graph0,{11270-11283},11283,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11270,11274,0x12c1f1c14000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11064,640_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op8,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op8,kernel_graph0,{11272},11272,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11272,11277,0x12c23a211600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11064,641_Default/learning_rate-CosineWithWarmUpLR/Add-op34,Default/learning_rate-CosineWithWarmUpLR/Add-op34,kernel_graph0,{11275},11275,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11275,11280,0x12c1f1c14000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11064,642_Default/optimizer-FP32StateAdamWeightDecay/Div-op8,Default/optimizer-FP32StateAdamWeightDecay/Div-op8,kernel_graph0,{11278},11278,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11278,11283,0x12c23a211600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11065,643_Default/optimizer-FP32StateAdamWeightDecay/Mul-op44,Default/optimizer-FP32StateAdamWeightDecay/Mul-op44,kernel_graph0,{11281},11281,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11281,11289,0x12c1f1c14000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11066,644_Default/optimizer-FP32StateAdamWeightDecay/Sub-op8,Default/optimizer-FP32StateAdamWeightDecay/Sub-op8,kernel_graph0,{11287},11287,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11290,11296,0x12c1e7936200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11069,648_Default/optimizer-FP32StateAdamWeightDecay/Mul-op45,Default/optimizer-FP32StateAdamWeightDecay/Mul-op45,kernel_graph0,{11294},11294,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11292,11300,0x12c23a211600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11069,649_Default/optimizer-FP32StateAdamWeightDecay/Mul-op47,Default/optimizer-FP32StateAdamWeightDecay/Mul-op47,kernel_graph0,{11298},11298,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11294,11319,0x12c1f1c14000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11070,650_Default/learning_rate-CosineWithWarmUpLR/Add-op35,Default/learning_rate-CosineWithWarmUpLR/Add-op35,kernel_graph0,{11307-11317},11317,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11298,11317,0x12c1f5c14200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11071,651_Default/learning_rate-CosineWithWarmUpLR/Add-op36,Default/learning_rate-CosineWithWarmUpLR/Add-op36,kernel_graph0,{11302-11315},11315,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11302,11306,0x12c1e7936200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11073,652_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op9,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op9,kernel_graph0,{11304},11304,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11304,11309,0x12c23a211600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11074,653_Default/learning_rate-CosineWithWarmUpLR/Add-op37,Default/learning_rate-CosineWithWarmUpLR/Add-op37,kernel_graph0,{11307},11307,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11307,11312,0x12c1e7936200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11075,654_Default/optimizer-FP32StateAdamWeightDecay/Div-op9,Default/optimizer-FP32StateAdamWeightDecay/Div-op9,kernel_graph0,{11310},11310,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11310,11315,0x12c23a211600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11077,655_Default/optimizer-FP32StateAdamWeightDecay/Mul-op49,Default/optimizer-FP32StateAdamWeightDecay/Mul-op49,kernel_graph0,{11313},11313,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11313,11321,0x12c1e7936200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11078,656_Default/optimizer-FP32StateAdamWeightDecay/Sub-op9,Default/optimizer-FP32StateAdamWeightDecay/Sub-op9,kernel_graph0,{11319},11319,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11322,11328,0x12c1e7936200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11081,660_Default/optimizer-FP32StateAdamWeightDecay/Mul-op50,Default/optimizer-FP32StateAdamWeightDecay/Mul-op50,kernel_graph0,{11326},11326,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11324,11332,0x12c23a211600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11081,661_Default/optimizer-FP32StateAdamWeightDecay/Mul-op52,Default/optimizer-FP32StateAdamWeightDecay/Mul-op52,kernel_graph0,{11330},11330,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11326,11351,0x12c1f1c14000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11083,662_Default/learning_rate-CosineWithWarmUpLR/Add-op38,Default/learning_rate-CosineWithWarmUpLR/Add-op38,kernel_graph0,{11339-11349},11349,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11330,11349,0x12c1f5c14200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11083,663_Default/learning_rate-CosineWithWarmUpLR/Add-op39,Default/learning_rate-CosineWithWarmUpLR/Add-op39,kernel_graph0,{11334-11347},11347,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11334,11338,0x12c23a211600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11084,664_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op10,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op10,kernel_graph0,{11336},11336,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11336,11341,0x12c1e3936000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11085,665_Default/learning_rate-CosineWithWarmUpLR/Add-op40,Default/learning_rate-CosineWithWarmUpLR/Add-op40,kernel_graph0,{11339},11339,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11339,11344,0x12c23a211600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11086,666_Default/optimizer-FP32StateAdamWeightDecay/Div-op10,Default/optimizer-FP32StateAdamWeightDecay/Div-op10,kernel_graph0,{11342},11342,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11342,11347,0x12c1e3936000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11087,667_Default/optimizer-FP32StateAdamWeightDecay/Mul-op54,Default/optimizer-FP32StateAdamWeightDecay/Mul-op54,kernel_graph0,{11345},11345,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11345,11353,0x12c23a211600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11088,668_Default/optimizer-FP32StateAdamWeightDecay/Sub-op10,Default/optimizer-FP32StateAdamWeightDecay/Sub-op10,kernel_graph0,{11351},11351,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11354,11360,0x12c23a211600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11092,672_Default/optimizer-FP32StateAdamWeightDecay/Mul-op55,Default/optimizer-FP32StateAdamWeightDecay/Mul-op55,kernel_graph0,{11358},11358,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11356,11364,0x12c1f1c14000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11092,673_Default/optimizer-FP32StateAdamWeightDecay/Mul-op57,Default/optimizer-FP32StateAdamWeightDecay/Mul-op57,kernel_graph0,{11362},11362,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11358,11383,0x12c1f5c14200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11092,674_Default/learning_rate-CosineWithWarmUpLR/Add-op41,Default/learning_rate-CosineWithWarmUpLR/Add-op41,kernel_graph0,{11371-11381},11381,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11362,11381,0x12c23a211600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11093,675_Default/learning_rate-CosineWithWarmUpLR/Add-op42,Default/learning_rate-CosineWithWarmUpLR/Add-op42,kernel_graph0,{11366-11379},11379,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11366,11370,0x12c1f1c14000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11094,676_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op11,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op11,kernel_graph0,{11368},11368,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11368,11373,0x12c204814600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11094,677_Default/learning_rate-CosineWithWarmUpLR/Add-op43,Default/learning_rate-CosineWithWarmUpLR/Add-op43,kernel_graph0,{11371},11371,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11371,11376,0x12c1f1c14000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11094,678_Default/optimizer-FP32StateAdamWeightDecay/Div-op11,Default/optimizer-FP32StateAdamWeightDecay/Div-op11,kernel_graph0,{11374},11374,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11374,11379,0x12c204814600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11095,679_Default/optimizer-FP32StateAdamWeightDecay/Mul-op59,Default/optimizer-FP32StateAdamWeightDecay/Mul-op59,kernel_graph0,{11377},11377,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11377,11385,0x12c1f1c14000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11096,680_Default/optimizer-FP32StateAdamWeightDecay/Sub-op11,Default/optimizer-FP32StateAdamWeightDecay/Sub-op11,kernel_graph0,{11383},11383,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11386,11392,0x12c204814600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11098,684_Default/optimizer-FP32StateAdamWeightDecay/Mul-op60,Default/optimizer-FP32StateAdamWeightDecay/Mul-op60,kernel_graph0,{11390},11390,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11388,11396,0x12c19f414400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11099,685_Default/optimizer-FP32StateAdamWeightDecay/Mul-op62,Default/optimizer-FP32StateAdamWeightDecay/Mul-op62,kernel_graph0,{11394},11394,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11390,11415,0x12c1aa014600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11099,686_Default/learning_rate-CosineWithWarmUpLR/Add-op44,Default/learning_rate-CosineWithWarmUpLR/Add-op44,kernel_graph0,{11403-11413},11413,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11394,11413,0x12c204814600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11100,687_Default/learning_rate-CosineWithWarmUpLR/Add-op45,Default/learning_rate-CosineWithWarmUpLR/Add-op45,kernel_graph0,{11398-11411},11411,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11398,11402,0x12c19f414400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11101,688_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op12,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op12,kernel_graph0,{11400},11400,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11400,11405,0x12c223621400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11101,689_Default/learning_rate-CosineWithWarmUpLR/Add-op46,Default/learning_rate-CosineWithWarmUpLR/Add-op46,kernel_graph0,{11403},11403,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11403,11408,0x12c19f414400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11102,690_Default/optimizer-FP32StateAdamWeightDecay/Div-op12,Default/optimizer-FP32StateAdamWeightDecay/Div-op12,kernel_graph0,{11406},11406,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11406,11411,0x12c223621400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11103,691_Default/optimizer-FP32StateAdamWeightDecay/Mul-op64,Default/optimizer-FP32StateAdamWeightDecay/Mul-op64,kernel_graph0,{11409},11409,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11409,11417,0x12c19f414400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11103,692_Default/optimizer-FP32StateAdamWeightDecay/Sub-op12,Default/optimizer-FP32StateAdamWeightDecay/Sub-op12,kernel_graph0,{11415},11415,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11418,11424,0x12c223621400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11110,696_Default/optimizer-FP32StateAdamWeightDecay/Mul-op65,Default/optimizer-FP32StateAdamWeightDecay/Mul-op65,kernel_graph0,{11422},11422,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11420,11428,0x12c204814600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11111,697_Default/optimizer-FP32StateAdamWeightDecay/Mul-op67,Default/optimizer-FP32StateAdamWeightDecay/Mul-op67,kernel_graph0,{11426},11426,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11422,11447,0x12c19f414400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11111,698_Default/learning_rate-CosineWithWarmUpLR/Add-op47,Default/learning_rate-CosineWithWarmUpLR/Add-op47,kernel_graph0,{11435-11445},11445,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11426,11445,0x12c223621400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11112,699_Default/learning_rate-CosineWithWarmUpLR/Add-op48,Default/learning_rate-CosineWithWarmUpLR/Add-op48,kernel_graph0,{11430-11443},11443,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11430,11434,0x12c22e225800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11113,700_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op13,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op13,kernel_graph0,{11432},11432,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11432,11437,0x12c204814600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11113,701_Default/learning_rate-CosineWithWarmUpLR/Add-op49,Default/learning_rate-CosineWithWarmUpLR/Add-op49,kernel_graph0,{11435},11435,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11435,11440,0x12c22e225800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11115,702_Default/optimizer-FP32StateAdamWeightDecay/Div-op13,Default/optimizer-FP32StateAdamWeightDecay/Div-op13,kernel_graph0,{11438},11438,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11438,11443,0x12c204814600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11116,703_Default/optimizer-FP32StateAdamWeightDecay/Mul-op69,Default/optimizer-FP32StateAdamWeightDecay/Mul-op69,kernel_graph0,{11441},11441,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11441,11449,0x12c22e225800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11116,704_Default/optimizer-FP32StateAdamWeightDecay/Sub-op13,Default/optimizer-FP32StateAdamWeightDecay/Sub-op13,kernel_graph0,{11447},11447,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11450,11456,0x12c223621400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11121,708_Default/optimizer-FP32StateAdamWeightDecay/Mul-op70,Default/optimizer-FP32StateAdamWeightDecay/Mul-op70,kernel_graph0,{11454},11454,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11452,11460,0x12c22e225800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11122,709_Default/optimizer-FP32StateAdamWeightDecay/Mul-op72,Default/optimizer-FP32StateAdamWeightDecay/Mul-op72,kernel_graph0,{11458},11458,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11454,11479,0x12c204814600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11122,710_Default/learning_rate-CosineWithWarmUpLR/Add-op50,Default/learning_rate-CosineWithWarmUpLR/Add-op50,kernel_graph0,{11467-11477},11477,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11458,11477,0x12c223621400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11123,711_Default/learning_rate-CosineWithWarmUpLR/Add-op51,Default/learning_rate-CosineWithWarmUpLR/Add-op51,kernel_graph0,{11462-11475},11475,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11462,11466,0x12c1f1c14000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11124,712_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op14,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op14,kernel_graph0,{11464},11464,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11464,11469,0x12c22e225800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11124,713_Default/learning_rate-CosineWithWarmUpLR/Add-op52,Default/learning_rate-CosineWithWarmUpLR/Add-op52,kernel_graph0,{11467},11467,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11467,11472,0x12c1f1c14000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11124,714_Default/optimizer-FP32StateAdamWeightDecay/Div-op14,Default/optimizer-FP32StateAdamWeightDecay/Div-op14,kernel_graph0,{11470},11470,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11470,11475,0x12c22e225800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11125,715_Default/optimizer-FP32StateAdamWeightDecay/Mul-op74,Default/optimizer-FP32StateAdamWeightDecay/Mul-op74,kernel_graph0,{11473},11473,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11473,11481,0x12c1f1c14000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11126,716_Default/optimizer-FP32StateAdamWeightDecay/Sub-op14,Default/optimizer-FP32StateAdamWeightDecay/Sub-op14,kernel_graph0,{11479},11479,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11482,11488,0x12c1f1c14000,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11128,720_Default/optimizer-FP32StateAdamWeightDecay/Mul-op75,Default/optimizer-FP32StateAdamWeightDecay/Mul-op75,kernel_graph0,{11486},11486,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11484,11492,0x12c19f414400,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11128,721_Default/optimizer-FP32StateAdamWeightDecay/Mul-op77,Default/optimizer-FP32StateAdamWeightDecay/Mul-op77,kernel_graph0,{11490},11490,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11486,11511,0x12c1be814600,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11128,722_Default/learning_rate-CosineWithWarmUpLR/Add-op53,Default/learning_rate-CosineWithWarmUpLR/Add-op53,kernel_graph0,{11499-11509},11509,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11490,11509,0x12c1f1c14000,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11130,723_Default/learning_rate-CosineWithWarmUpLR/Add-op54,Default/learning_rate-CosineWithWarmUpLR/Add-op54,kernel_graph0,{11494-11507},11507,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11494,11498,0x12c180014200,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11131,724_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op15,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op15,kernel_graph0,{11496},11496,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11496,11501,0x12c19f414400,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11131,725_Default/learning_rate-CosineWithWarmUpLR/Add-op55,Default/learning_rate-CosineWithWarmUpLR/Add-op55,kernel_graph0,{11499},11499,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11499,11504,0x12c180014200,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11132,726_Default/optimizer-FP32StateAdamWeightDecay/Div-op15,Default/optimizer-FP32StateAdamWeightDecay/Div-op15,kernel_graph0,{11502},11502,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11502,11507,0x12c19f414400,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11133,727_Default/optimizer-FP32StateAdamWeightDecay/Mul-op79,Default/optimizer-FP32StateAdamWeightDecay/Mul-op79,kernel_graph0,{11505},11505,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11505,11513,0x12c180014200,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11133,728_Default/optimizer-FP32StateAdamWeightDecay/Sub-op15,Default/optimizer-FP32StateAdamWeightDecay/Sub-op15,kernel_graph0,{11511},11511,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11514,11520,0x12c223621400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11138,732_Default/optimizer-FP32StateAdamWeightDecay/Mul-op80,Default/optimizer-FP32StateAdamWeightDecay/Mul-op80,kernel_graph0,{11518},11518,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11516,11524,0x12c223625600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11139,733_Default/optimizer-FP32StateAdamWeightDecay/Mul-op82,Default/optimizer-FP32StateAdamWeightDecay/Mul-op82,kernel_graph0,{11522},11522,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11518,11543,0x12c223629800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11139,734_Default/learning_rate-CosineWithWarmUpLR/Add-op56,Default/learning_rate-CosineWithWarmUpLR/Add-op56,kernel_graph0,{11531-11541},11541,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11522,11541,0x12c22361d200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11142,735_Default/learning_rate-CosineWithWarmUpLR/Add-op57,Default/learning_rate-CosineWithWarmUpLR/Add-op57,kernel_graph0,{11526-11539},11539,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11526,11530,0x12c223621400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11143,736_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op16,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op16,kernel_graph0,{11528},11528,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11528,11533,0x12c223625600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11143,737_Default/learning_rate-CosineWithWarmUpLR/Add-op58,Default/learning_rate-CosineWithWarmUpLR/Add-op58,kernel_graph0,{11531},11531,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11531,11536,0x12c223621400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11144,738_Default/optimizer-FP32StateAdamWeightDecay/Div-op16,Default/optimizer-FP32StateAdamWeightDecay/Div-op16,kernel_graph0,{11534},11534,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11534,11539,0x12c223625600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11145,739_Default/optimizer-FP32StateAdamWeightDecay/Mul-op84,Default/optimizer-FP32StateAdamWeightDecay/Mul-op84,kernel_graph0,{11537},11537,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11537,11545,0x12c223621400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11145,740_Default/optimizer-FP32StateAdamWeightDecay/Sub-op16,Default/optimizer-FP32StateAdamWeightDecay/Sub-op16,kernel_graph0,{11543},11543,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11546,11552,0x12c22361d200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11150,744_Default/optimizer-FP32StateAdamWeightDecay/Mul-op85,Default/optimizer-FP32StateAdamWeightDecay/Mul-op85,kernel_graph0,{11550},11550,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11548,11556,0x12c223621400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11151,745_Default/optimizer-FP32StateAdamWeightDecay/Mul-op87,Default/optimizer-FP32StateAdamWeightDecay/Mul-op87,kernel_graph0,{11554},11554,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11550,11575,0x12c223625600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11151,746_Default/learning_rate-CosineWithWarmUpLR/Add-op59,Default/learning_rate-CosineWithWarmUpLR/Add-op59,kernel_graph0,{11563-11573},11573,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11554,11573,0x12c223619000,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11153,747_Default/learning_rate-CosineWithWarmUpLR/Add-op60,Default/learning_rate-CosineWithWarmUpLR/Add-op60,kernel_graph0,{11558-11571},11571,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11558,11562,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11154,748_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op17,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op17,kernel_graph0,{11560},11560,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11560,11565,0x12c22361d200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11154,749_Default/learning_rate-CosineWithWarmUpLR/Add-op61,Default/learning_rate-CosineWithWarmUpLR/Add-op61,kernel_graph0,{11563},11563,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11563,11568,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11155,750_Default/optimizer-FP32StateAdamWeightDecay/Div-op17,Default/optimizer-FP32StateAdamWeightDecay/Div-op17,kernel_graph0,{11566},11566,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11566,11571,0x12c22361d200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11156,751_Default/optimizer-FP32StateAdamWeightDecay/Mul-op89,Default/optimizer-FP32StateAdamWeightDecay/Mul-op89,kernel_graph0,{11569},11569,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11569,11577,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11156,752_Default/optimizer-FP32StateAdamWeightDecay/Sub-op17,Default/optimizer-FP32StateAdamWeightDecay/Sub-op17,kernel_graph0,{11575},11575,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11578,11584,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11158,756_Default/optimizer-FP32StateAdamWeightDecay/Mul-op90,Default/optimizer-FP32StateAdamWeightDecay/Mul-op90,kernel_graph0,{11582},11582,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11580,11588,0x12c223619000,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11158,757_Default/optimizer-FP32StateAdamWeightDecay/Mul-op92,Default/optimizer-FP32StateAdamWeightDecay/Mul-op92,kernel_graph0,{11586},11586,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11582,11607,0x12c22361d200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11159,758_Default/learning_rate-CosineWithWarmUpLR/Add-op62,Default/learning_rate-CosineWithWarmUpLR/Add-op62,kernel_graph0,{11595-11605},11605,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11586,11605,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11160,759_Default/learning_rate-CosineWithWarmUpLR/Add-op63,Default/learning_rate-CosineWithWarmUpLR/Add-op63,kernel_graph0,{11590-11603},11603,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11590,11594,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11160,760_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op18,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op18,kernel_graph0,{11592},11592,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11592,11597,0x12c223621400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11160,761_Default/learning_rate-CosineWithWarmUpLR/Add-op64,Default/learning_rate-CosineWithWarmUpLR/Add-op64,kernel_graph0,{11595},11595,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11595,11600,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11162,762_Default/optimizer-FP32StateAdamWeightDecay/Div-op18,Default/optimizer-FP32StateAdamWeightDecay/Div-op18,kernel_graph0,{11598},11598,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11598,11603,0x12c223621400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11163,763_Default/optimizer-FP32StateAdamWeightDecay/Mul-op94,Default/optimizer-FP32StateAdamWeightDecay/Mul-op94,kernel_graph0,{11601},11601,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11601,11609,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11163,764_Default/optimizer-FP32StateAdamWeightDecay/Sub-op18,Default/optimizer-FP32StateAdamWeightDecay/Sub-op18,kernel_graph0,{11607},11607,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11610,11616,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11166,768_Default/optimizer-FP32StateAdamWeightDecay/Mul-op97,Default/optimizer-FP32StateAdamWeightDecay/Mul-op97,kernel_graph0,{11614},11614,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11612,11620,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11167,769_Default/optimizer-FP32StateAdamWeightDecay/Mul-op95,Default/optimizer-FP32StateAdamWeightDecay/Mul-op95,kernel_graph0,{11618},11618,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11614,11637,0x12c1f1c14000,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11167,770_Default/learning_rate-CosineWithWarmUpLR/Add-op66,Default/learning_rate-CosineWithWarmUpLR/Add-op66,kernel_graph0,{11622-11635},11635,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11618,11639,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11170,771_Default/learning_rate-CosineWithWarmUpLR/Add-op65,Default/learning_rate-CosineWithWarmUpLR/Add-op65,kernel_graph0,{11627-11637},11637,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11622,11626,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11171,772_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op19,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op19,kernel_graph0,{11624},11624,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11624,11629,0x12c1f1c18200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11171,773_Default/learning_rate-CosineWithWarmUpLR/Add-op67,Default/learning_rate-CosineWithWarmUpLR/Add-op67,kernel_graph0,{11627},11627,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11627,11632,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11174,774_Default/optimizer-FP32StateAdamWeightDecay/Div-op19,Default/optimizer-FP32StateAdamWeightDecay/Div-op19,kernel_graph0,{11630},11630,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11630,11635,0x12c1f1c18200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11175,775_Default/optimizer-FP32StateAdamWeightDecay/Mul-op99,Default/optimizer-FP32StateAdamWeightDecay/Mul-op99,kernel_graph0,{11633},11633,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11633,11641,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11175,776_Default/optimizer-FP32StateAdamWeightDecay/Sub-op19,Default/optimizer-FP32StateAdamWeightDecay/Sub-op19,kernel_graph0,{11639},11639,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11642,11648,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11179,780_Default/optimizer-FP32StateAdamWeightDecay/Mul-op102,Default/optimizer-FP32StateAdamWeightDecay/Mul-op102,kernel_graph0,{11646},11646,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11644,11652,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11180,781_Default/optimizer-FP32StateAdamWeightDecay/Mul-op100,Default/optimizer-FP32StateAdamWeightDecay/Mul-op100,kernel_graph0,{11650},11650,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11646,11674,0x12c180010000,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11180,782_Default/learning_rate-CosineWithWarmUpLR/Add-op69,Default/learning_rate-CosineWithWarmUpLR/Add-op69,kernel_graph0,{11654-11672},11672,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11650,11672,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11182,783_Default/learning_rate-CosineWithWarmUpLR/Add-op68,Default/learning_rate-CosineWithWarmUpLR/Add-op68,kernel_graph0,{11659-11670},11670,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11654,11658,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11183,784_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op20,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op20,kernel_graph0,{11656},11656,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11656,11661,0x12c18000be00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11183,785_Default/learning_rate-CosineWithWarmUpLR/Add-op70,Default/learning_rate-CosineWithWarmUpLR/Add-op70,kernel_graph0,{11659},11659,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11659,11665,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11185,786_Default/optimizer-FP32StateAdamWeightDecay/Div-op20,Default/optimizer-FP32StateAdamWeightDecay/Div-op20,kernel_graph0,{11662},11662,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11662,11668,0x12c18000be00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11186,787_Default/optimizer-FP32StateAdamWeightDecay/Mul-op104,Default/optimizer-FP32StateAdamWeightDecay/Mul-op104,kernel_graph0,{11666},11666,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11666,11670,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11186,788_Default/optimizer-FP32StateAdamWeightDecay/Sub-op20,Default/optimizer-FP32StateAdamWeightDecay/Sub-op20,kernel_graph0,{11668},11668,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11675,13974,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11188,792_Default/TensorMove-op2,Default/TensorMove-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11679,13972,0x12c75d35ec00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/output_actor.cc,689,Other,11677,kernel_graph_0_OutputActor,UpdateOutputDeviceAddress,,{},,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 11685,12089,0x12c75d35ee00,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Kernel,11684,0_Default/network-LlamaForCausalLM/StridedSlice-op0,inputs0,0_Default/network-LlamaForCausalLM/StridedSlice-op0,{11686-12087},12087,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,1,0, 11686,12668,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11684,0_Default/network-LlamaForCausalLM/StridedSlice-op0,inputs0,0_Default/network-LlamaForCausalLM/StridedSlice-op0,{11688-11690-12665},12665,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11688,11694,0x12c18000be00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11686,1_Default/network-LlamaForCausalLM/model-LlamaModel/tok_embeddings-LlamaEmbedding/Gather-op0,Default/network-LlamaForCausalLM/model-LlamaModel/tok_embeddings-LlamaEmbedding/Gather-op0,kernel_graph0,{11692},11692,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11690,11700,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,8704,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11687,2_Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/NotEqual-op0,Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/NotEqual-op0,kernel_graph0,{11695-11698},11698,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11692,12646,0x12c18800c000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11687,3_Default/network-LlamaForCausalLM/model-LlamaModel/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/Cast-op0,kernel_graph0,{11704-11833-12644},12644,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11695,12123,0x12c18000be00,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11687,4_Default/network-LlamaForCausalLM/Cast-op1,Default/network-LlamaForCausalLM/Cast-op1,kernel_graph0,{11696-11715-12076-12120},12120,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11698,11729,0x12c180014000,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11687,6_Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Cast-op0,kernel_graph0,{11723-11727},11727,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11701,12572,0x12c180018200,0,DefaultEnhancedAscendMemoryPool,524800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11688,7_Default/network-LlamaForCausalLM/model-LlamaModel/freqs_mgr-FreqsMgr/StridedSlice-op0,Default/network-LlamaForCausalLM/model-LlamaModel/freqs_mgr-FreqsMgr/StridedSlice-op0,kernel_graph0,{11702-11766-11789-11919-11934-12307-12328-12549-12570},12570,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11704,11710,0x12c18c00c200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11689,9_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op0,kernel_graph0,{11708},11708,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11706,11714,0x12c19400c400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11689,10_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{11712},11712,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11707,12653,0x12c180098400,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11689,10_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{12649},12649,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11708,11711,0x12c1800a0600,0,DefaultEnhancedAscendMemoryPool,16777728,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11689,10_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11712,12624,0x12c1800a0600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11690,11_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op1,kernel_graph0,{11737-11742-11757-11780-12594-12608-12622},12622,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11715,11723,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11691,12_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/ReduceSum-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/ReduceSum-op0,kernel_graph0,{11721},11721,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11717,11719,0x12c180008800,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,11717,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11718,11720,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,11718,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11721,12134,0x12c180008800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11691,13_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Add-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Add-op0,kernel_graph0,{12064-12131},12131,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11725,11730,0x12c1840a0800,0,DefaultEnhancedAscendMemoryPool,2097664,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11693,15_Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/ExpandDims-op0,Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/ExpandDims-op0,kernel_graph0,{11727},11727,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11727,11733,0x12c1842a0a00,0,DefaultEnhancedAscendMemoryPool,16777728,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11693,16_Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Mul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Mul-op0,kernel_graph0,{11731},11731,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11731,11737,0x12c1852a0c00,0,DefaultEnhancedAscendMemoryPool,16777728,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11695,17_Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Sub-op0,Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Sub-op0,kernel_graph0,{11733-11735},11735,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11735,12530,0x12c1840a0800,0,DefaultEnhancedAscendMemoryPool,8389120,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11696,19_Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Cast-op1,kernel_graph0,{11818-11965-12283-12524},12524,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11739,12618,0x12c1848a0a00,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11697,21_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/Cast-op0,kernel_graph0,{11742-12615},12615,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11741,11749,0x12c18c00c200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11698,22_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,kernel_graph0,{11744-11747},11747,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11742,11744,0x12c1868a0c00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11698,22_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11746,11753,0x12c19000c400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11700,24_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,kernel_graph0,{11751},11751,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11747,11750,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11700,24_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11751,11770,0x12c19400c600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11701,25_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op2,kernel_graph0,{11766-11768},11768,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11754,12633,0x12c18c00c200,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11702,26_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/Cast-op0,kernel_graph0,{11757-12630},12630,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11756,11764,0x12c18e00c400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11702,27_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,kernel_graph0,{11759-11762},11762,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11757,11759,0x12c1868a0c00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11702,27_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11761,11773,0x12c19c00c800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11702,29_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,kernel_graph0,{11771},11771,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11762,11765,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11702,29_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11766,11809,0x12c1a000ca00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11702,30_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op2,kernel_graph0,{11807},11807,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11768,11806,0x12c1a800cc00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11703,31_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op1,kernel_graph0,{11804},11804,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11771,11793,0x12c18e00c400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11704,32_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op0,kernel_graph0,{11789-11791},11791,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11774,12575,0x12c1868a0c00,0,DefaultEnhancedAscendMemoryPool,524800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11705,33_Default/network-LlamaForCausalLM/model-LlamaModel/freqs_mgr-FreqsMgr/StridedSlice-op1,Default/network-LlamaForCausalLM/model-LlamaModel/freqs_mgr-FreqsMgr/StridedSlice-op1,kernel_graph0,{11775-11794-11804-11924-11939-12309-12330-12551-12573},12573,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11777,12604,0x12c19600c600,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11708,35_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/Cast-op0,kernel_graph0,{11780-12601},12601,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11779,11787,0x12c19800c800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11709,36_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,kernel_graph0,{11782-11785},11785,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11780,11782,0x12c186920e00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11709,36_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11784,12528,0x12c1b000ce00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11712,38_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,kernel_graph0,{11818-12524},12524,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11785,11788,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11712,38_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11789,11799,0x12c19800c800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11713,39_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op0,kernel_graph0,{11797},11797,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11791,11796,0x12c1b400d000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11713,40_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op0,kernel_graph0,{11794},11794,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11794,11800,0x12c18e00c400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11715,41_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op1,kernel_graph0,{11797},11797,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11797,11803,0x12c1b400d000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11716,42_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op0,kernel_graph0,{11801},11801,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11801,12526,0x12c18e00c400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11716,43_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op1,kernel_graph0,{11818-12524},12524,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11804,11810,0x12c19800c800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11716,44_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op3,kernel_graph0,{11807},11807,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11807,11813,0x12c1a800cc00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11716,45_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op1,kernel_graph0,{11811},11811,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11811,12527,0x12c19800c800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11717,46_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op3,kernel_graph0,{11818-12524},12524,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11814,12531,0x12c186920e00,0,DefaultEnhancedAscendMemoryPool,8389120,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11718,47_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{12524},12524,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11815,12532,0x12c187121000,0,DefaultEnhancedAscendMemoryPool,8389120,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11718,47_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{12524},12524,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11816,12533,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11718,47_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{12524},12524,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11817,12534,0x12c19c00ca00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11718,47_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{11822-12524},12524,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11818,11820,0x12c1a000cc00,0,DefaultEnhancedAscendMemoryPool,84936192,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11718,47_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11821,12504,0x12c1a000cc00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11718,48_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,kernel_graph0,{11824-11829-12502},12502,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11822,11824,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11718,48_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11826,12512,0x12c19200c600,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11721,50_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/Cast-op0,kernel_graph0,{11829-12510},12510,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11828,11835,0x12c1a400ce00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11722,51_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,kernel_graph0,{11831-11833},11833,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11829,11831,0x12c19400c800,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11722,51_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11833,12484,0x12c1a800d000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11723,53_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/Add-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/Add-op0,kernel_graph0,{11836-11874-12482},12482,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11836,11842,0x12c1b400d000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11724,54_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op0,kernel_graph0,{11840},11840,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11838,11846,0x12c1bc00d200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11725,55_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{11844},11844,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11839,12491,0x12c187921200,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11725,55_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{12487},12487,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11840,11843,0x12c19400c800,0,DefaultEnhancedAscendMemoryPool,16777728,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11725,55_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11844,12463,0x12c1a400ce00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11726,56_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op1,kernel_graph0,{11846-11851-11858-12447-12461},12461,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11848,12457,0x12c1b400d000,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11727,58_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/Cast-op0,kernel_graph0,{11851-12454},12454,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11850,12439,0x12c1b960d200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11728,59_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,kernel_graph0,{11853-11864-12437},12437,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11851,11853,0x12c19400c800,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11728,59_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11855,12472,0x12c1c420d400,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11731,61_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/Cast-op0,kernel_graph0,{11858-12469},12469,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11857,12445,0x12c1c980d600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11732,62_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,kernel_graph0,{11861-12442},12442,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11858,11860,0x12c19400c800,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11732,62_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11861,12435,0x12c1d440d800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11733,63_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/SiLU-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/SiLU-op0,kernel_graph0,{11862-11864-12433},12433,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11864,12422,0x12c1df00da00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11735,65_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/Mul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/Mul-op0,kernel_graph0,{11865-11870-12420},12420,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11867,12430,0x12c1e9c0dc00,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11736,67_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/Cast-op0,kernel_graph0,{11870-12428},12428,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11869,11876,0x12c1ef20de00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11737,68_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,kernel_graph0,{11872-11874},11874,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11870,11872,0x12c19400c800,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11737,68_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11874,12402,0x12c1f320e000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11739,70_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/Add-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/Add-op1,kernel_graph0,{11877-11980-12400},12400,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11877,11883,0x12c1f720e200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11740,71_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op0,kernel_graph0,{11881},11881,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11879,11887,0x12c1ff20e400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11741,72_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{11885},11885,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11880,12409,0x12c187929400,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11741,72_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{12405},12405,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11881,11884,0x12c19400c800,0,DefaultEnhancedAscendMemoryPool,16777728,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11741,72_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11885,12380,0x12c1ef20de00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11742,73_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op1,kernel_graph0,{11887-11892-11907-11952-12350-12364-12378},12378,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11889,12374,0x12c1ac00d200,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11743,75_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/Cast-op0,kernel_graph0,{11892-12371},12371,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11891,11899,0x12c1f720e200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11744,76_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,kernel_graph0,{11894-11897},11897,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11892,11894,0x12c1ae00d400,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11744,76_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11896,11903,0x12c1fb20e400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11747,78_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,kernel_graph0,{11901},11901,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11897,11900,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11747,78_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11901,11923,0x12c1ff20e600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11748,79_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op2,kernel_graph0,{11919-11921},11921,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11904,12389,0x12c1f720e200,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11748,80_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/Cast-op0,kernel_graph0,{11907-12386},12386,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11906,11914,0x12c1f920e400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11751,81_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,kernel_graph0,{11909-11912},11912,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11907,11909,0x12c1ae00d400,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11751,81_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11911,11918,0x12c20720e800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11752,83_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,kernel_graph0,{11916},11916,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11912,11915,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11752,83_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11916,11938,0x12c20b20ea00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11754,84_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op0,kernel_graph0,{11934-11936},11936,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11919,11929,0x12c21320ec00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11755,85_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op2,kernel_graph0,{11927},11927,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11921,11926,0x12c21b20ee00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11755,86_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op1,kernel_graph0,{11924},11924,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11924,11930,0x12c1f920e400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11757,87_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op3,kernel_graph0,{11927},11927,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11927,11933,0x12c20120e600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11757,88_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op1,kernel_graph0,{11931},11931,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11931,12286,0x12c1f920e400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11758,89_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op3,kernel_graph0,{11965-12283},12283,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11934,11944,0x12c1fd20e600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11758,90_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op0,kernel_graph0,{11942},11942,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11936,11941,0x12c21320ec00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11759,91_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op0,kernel_graph0,{11939},11939,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11939,11945,0x12c20520e800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11760,92_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op1,kernel_graph0,{11942},11942,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11942,11948,0x12c20d20ea00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11762,93_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op0,kernel_graph0,{11946},11946,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11946,12285,0x12c1fd20e600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11763,94_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op1,kernel_graph0,{11965-12283},12283,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11949,12360,0x12c20120e800,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11763,95_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/Cast-op0,kernel_graph0,{11952-12357},12357,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11951,11959,0x12c20320ea00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11766,96_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,kernel_graph0,{11954-11957},11957,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11952,11954,0x12c1ae00d400,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11766,96_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11956,12287,0x12c20720ec00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11767,98_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,kernel_graph0,{11965-12283},12283,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11957,11960,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11767,98_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11961,12289,0x12c1ae00d400,0,DefaultEnhancedAscendMemoryPool,8389120,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11768,99_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{12283},12283,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11962,12290,0x12c1ae80d600,0,DefaultEnhancedAscendMemoryPool,8389120,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11768,99_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{12283},12283,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11963,12291,0x12c180008a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11768,99_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{12283},12283,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11964,12292,0x12c20320ea00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11768,99_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{11969-12283},12283,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11965,11967,0x12c20b20ee00,0,DefaultEnhancedAscendMemoryPool,84936192,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11768,99_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11968,12263,0x12c20b20ee00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11769,100_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,kernel_graph0,{11971-11976-12261},12261,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11969,11971,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11769,100_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11973,12271,0x12c20f20f000,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11770,102_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/Cast-op0,kernel_graph0,{11976-12269},12269,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11975,11982,0x12c21120f200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11771,103_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,kernel_graph0,{11978-11980},11980,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11976,11978,0x12c19400c800,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11771,103_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11980,12238,0x12c21520f400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11772,105_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/Add-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/Add-op0,kernel_graph0,{11983-12021-12236},12236,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11983,11989,0x12c21920f600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11774,106_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op0,kernel_graph0,{11987},11987,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11985,11993,0x12c22120f800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11775,107_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{11991},11991,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11986,12245,0x12c187931600,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11775,107_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{12241},12241,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11987,11990,0x12c19400c800,0,DefaultEnhancedAscendMemoryPool,16777728,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11775,107_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11991,12217,0x12c21120f200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11775,108_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op1,kernel_graph0,{11993-11998-12005-12201-12215},12215,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11995,12211,0x12c21920f600,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11775,110_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/Cast-op0,kernel_graph0,{11998-12208},12208,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11997,12193,0x12c21e80f800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11775,111_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,kernel_graph0,{12000-12011-12191},12191,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 11998,12000,0x12c19400c800,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11775,111_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12002,12226,0x12c22940fa00,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11777,113_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/Cast-op0,kernel_graph0,{12005-12223},12223,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12004,12199,0x12c22ea0fc00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11778,114_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,kernel_graph0,{12008-12196},12196,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12005,12007,0x12c19400c800,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11778,114_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12008,12189,0x12c23960fe00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11778,115_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/SiLU-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/SiLU-op0,kernel_graph0,{12009-12011-12187},12187,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12011,12176,0x12c244210000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11781,117_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/Mul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/Mul-op0,kernel_graph0,{12012-12017-12174},12174,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12014,12184,0x12c24ee10200,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11782,119_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/Cast-op0,kernel_graph0,{12017-12182},12182,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12016,12023,0x12c254410400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11783,120_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,kernel_graph0,{12019-12021},12021,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12017,12019,0x12c19400c800,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11783,120_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12021,12155,0x12c258410600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11786,122_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/Add-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/Add-op1,kernel_graph0,{12024-12153},12153,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12024,12030,0x12c25c410800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11786,123_Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Cast-op0,kernel_graph0,{12028},12028,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12026,12034,0x12c264410a00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11789,124_Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{12032},12032,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12027,12162,0x12c187939800,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11789,124_Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{12158},12158,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12028,12031,0x12c19400c800,0,DefaultEnhancedAscendMemoryPool,16777728,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11789,124_Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12032,12138,0x12c254410400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11790,125_Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Cast-op1,kernel_graph0,{12034-12039-12136},12136,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12036,12147,0x12c25c410800,0,DefaultEnhancedAscendMemoryPool,262144512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11791,127_Default/network-LlamaForCausalLM/lm_head-Linear/Cast-op0,Default/network-LlamaForCausalLM/lm_head-Linear/Cast-op0,kernel_graph0,{12039-12144},12144,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12038,12044,0x12c26be10a00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11791,128_Default/network-LlamaForCausalLM/lm_head-Linear/MatMul-op0,Default/network-LlamaForCausalLM/lm_head-Linear/MatMul-op0,kernel_graph0,{12042},12042,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12039,12041,0x12c19400c800,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11791,128_Default/network-LlamaForCausalLM/lm_head-Linear/MatMul-op0,Default/network-LlamaForCausalLM/lm_head-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12042,12051,0x12c28b210c00,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11792,129_Default/network-LlamaForCausalLM/Cast-op0,Default/network-LlamaForCausalLM/Cast-op0,kernel_graph0,{12046-12049},12049,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12045,12052,0x12c187941a00,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11793,130_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/ReduceMax-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/ReduceMax-op0,kernel_graph0,{12049},12049,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12046,12048,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11793,130_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/ReduceMax-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/ReduceMax-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12049,12085,0x12c2c9a10e00,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11794,131_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Sub-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Sub-op0,kernel_graph0,{12055-12083},12083,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12053,12066,0x12c180008c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11795,132_Default/Mul-op0,Default/Mul-op0,kernel_graph0,{12064},12064,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12055,12063,0x12c26be10a00,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11797,133_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Exp-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Exp-op0,kernel_graph0,{12057},12057,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12057,12082,0x12c187941a00,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11798,134_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/ReduceSum-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/ReduceSum-op0,kernel_graph0,{12080},12080,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12059,12061,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,12059,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12060,12062,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,12060,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12064,12070,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11798,135_Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_RealDiv/RealDiv-op0,Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_RealDiv/RealDiv-op0,kernel_graph0,{12068},12068,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12067,12075,0x12c180008c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11801,136_Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_RealDiv/SumExt-op0,Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_RealDiv/SumExt-op0,kernel_graph0,{12071-12073},12073,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12068,12071,0x12c180009600,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11801,136_Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_RealDiv/SumExt-op0,Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_RealDiv/SumExt-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12073,12078,0x12c187949c00,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11802,138_Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_ReduceSum/Tile-op0,Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_ReduceSum/Tile-op0,kernel_graph0,{12076},12076,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12076,12102,0x12c187951e00,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11803,139_Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_Mul/Mul-op0,Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_Mul/Mul-op0,kernel_graph0,{12078-12099},12099,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12080,12086,0x12c187949c00,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11805,141_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Log-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Log-op0,kernel_graph0,{12083},12083,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12083,12108,0x12c26be10a00,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11806,142_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Sub-op1,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Sub-op1,kernel_graph0,{12094-12106},12106,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12087,12093,0x12c187941a00,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11807,143_Default/network-LlamaForCausalLM/StridedSlice-op1,Default/network-LlamaForCausalLM/StridedSlice-op1,kernel_graph0,{12089-12091},12091,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12091,12109,0x12c2aa610c00,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11810,145_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/OneHot-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/OneHot-op0,kernel_graph0,{12096-12106},12106,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12094,12098,0x12c2e8e10e00,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11811,146_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Exp-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Exp-op0,kernel_graph0,{12096},12096,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12096,12101,0x12c327611000,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11812,147_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Sub-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Sub-op0,kernel_graph0,{12099},12099,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12099,12105,0x12c2e8e10e00,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11812,148_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Mul-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Mul-op0,kernel_graph0,{12103},12103,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12103,12146,0x12c327611000,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11818,149_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op0,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op0,kernel_graph0,{12136-12144},12144,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12106,12112,0x12c2e8e10e00,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11818,150_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Mul-op1,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Mul-op1,kernel_graph0,{12110},12110,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12110,12119,0x12c26be10a00,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11819,151_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Neg-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Neg-op0,kernel_graph0,{12113},12113,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12113,12122,0x12c187941a00,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11819,152_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/ReduceSum-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/ReduceSum-op0,kernel_graph0,{12120},12120,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12115,12117,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,12115,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12116,12118,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,12116,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12120,12130,0x12c187949c00,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11819,153_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Mul-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Mul-op0,kernel_graph0,{12124},12124,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12124,12133,0x12c180008c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11822,154_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/ReduceSum-op1,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/ReduceSum-op1,kernel_graph0,{12131},12131,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12126,12128,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,12126,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12127,12129,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,12127,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12131,13967,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11822,155_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/RealDiv-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/RealDiv-op0,kernel_graph0,{13965},13965,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12135,12142,0x12c346a11200,0,DefaultEnhancedAscendMemoryPool,262144512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11823,156_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op29,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op29,kernel_graph0,{12140},12140,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12136,12139,0x12c19400c800,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11823,156_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op29,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op29,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12140,12721,0x12c26be10a00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11823,157_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op29,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op29,kernel_graph0,{12719},12719,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12143,12152,0x12c254410400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11824,158_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op0,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op0,kernel_graph0,{12148-12150},12150,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12144,12148,0x12c346a11200,0,DefaultEnhancedAscendMemoryPool,155190272,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11824,158_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op0,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12150,12160,0x12c25c410800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11826,160_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op1,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op1,kernel_graph0,{12158},12158,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12153,12161,0x12c28b210c00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11827,161_Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Cast-op2,kernel_graph0,{12158},12158,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12156,12166,0x12c254410400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11827,162_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op0,kernel_graph0,{12164},12164,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12157,12736,0x12c18000be00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11827,162_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op0,kernel_graph0,{12734},12734,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12158,12163,0x12c19400c800,0,DefaultEnhancedAscendMemoryPool,16778752,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11827,162_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12164,12253,0x12c25c410600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11829,163_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op0,kernel_graph0,{12168-12174-12182-12250},12250,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12167,12172,0x12c180008800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11830,164_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Mul-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Mul-op0,kernel_graph0,{12170},12170,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12170,12936,0x12c180008c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11831,166_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Maximum-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Maximum-op0,kernel_graph0,{12934},12934,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12173,12180,0x12c254410400,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11831,167_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op27,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op27,kernel_graph0,{12178},12178,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12174,12177,0x12c19400c800,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11831,167_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op27,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op27,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12178,12715,0x12c244210000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11832,168_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op27,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op27,kernel_graph0,{12713},12713,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12181,12194,0x12c260410800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11833,169_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op1,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op1,kernel_graph0,{12185-12187-12191},12191,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12182,12185,0x12c19400c800,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11833,169_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op1,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12187,12210,0x12c24ee10200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11836,171_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op0,kernel_graph0,{12189-12201-12208},12208,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12191,12198,0x12c23960fe00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11837,173_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op1,kernel_graph0,{12194-12196},12196,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12196,12225,0x12c21e80f800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11840,175_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/Grad_SiLU/SiLUGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/Grad_SiLU/SiLUGrad-op0,kernel_graph0,{12215-12223},12223,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12200,12206,0x12c260410800,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11841,176_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op28,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op28,kernel_graph0,{12204},12204,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12201,12203,0x12c19400c800,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11841,176_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op28,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op28,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12204,12718,0x12c22ea0fc00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11841,177_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op28,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op28,kernel_graph0,{12716},12716,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12207,12231,0x12c23960fe00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11844,178_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op2,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op2,kernel_graph0,{12212-12229},12229,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12208,12212,0x12c19400c800,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11844,178_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op2,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12214,12221,0x12c21920f600,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11845,180_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op26,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op26,kernel_graph0,{12219},12219,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12215,12218,0x12c19400c800,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11845,180_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op26,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op26,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12219,12712,0x12c260410800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11846,181_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op26,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op26,kernel_graph0,{12710},12710,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12222,12232,0x12c21120f200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11847,182_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op3,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op3,kernel_graph0,{12227-12229},12229,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12223,12227,0x12c19400c800,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11847,182_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op3,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op3,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12229,12235,0x12c23d610000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11849,184_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/AddN-op0,kernel_graph0,{12233},12233,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12233,12243,0x12c24ee10200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11849,185_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op2,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op2,kernel_graph0,{12241},12241,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12236,12244,0x12c23960fe00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11851,186_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op2,kernel_graph0,{12241},12241,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12239,12249,0x12c21120f200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11852,187_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op1,kernel_graph0,{12247},12247,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12240,12730,0x12c180010000,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11852,187_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op1,kernel_graph0,{12728},12728,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12241,12246,0x12c19400c800,0,DefaultEnhancedAscendMemoryPool,16778752,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11852,187_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12247,12252,0x12c23960fe00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11852,188_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op1,kernel_graph0,{12250},12250,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12250,12417,0x12c23d610000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11853,189_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/AddN-op0,kernel_graph0,{12255-12261-12269-12414},12414,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12254,12259,0x12c180008800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11853,190_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Sub-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Sub-op0,kernel_graph0,{12257},12257,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12257,12959,0x12c180009600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11855,192_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/LessEqual-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/LessEqual-op0,kernel_graph0,{12939-12957},12957,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12260,12267,0x12c241610200,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11856,193_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op25,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op25,kernel_graph0,{12265},12265,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12261,12264,0x12c19400c800,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11856,193_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op25,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op25,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12265,12709,0x12c20b20ee00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11856,194_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op25,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op25,kernel_graph0,{12707},12707,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12268,12277,0x12c23960fe00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11858,195_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op4,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op4,kernel_graph0,{12272-12275},12275,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12269,12272,0x12c19400c800,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11858,195_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op4,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op4,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12274,12288,0x12c24ee10200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11859,197_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op0,kernel_graph0,{12283},12283,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12275,12278,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11859,197_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12279,12300,0x12c23960fe00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11861,198_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,kernel_graph0,{12298},12298,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12280,12297,0x12c252e10400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11861,198_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,kernel_graph0,{12295},12295,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12281,12304,0x12c256e10600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11861,198_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,kernel_graph0,{12302},12302,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12282,12293,0x12c180008800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11861,198_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12283,12294,0x12c20f20f000,0,DefaultEnhancedAscendMemoryPool,515969024,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11861,198_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12295,12311,0x12c1f920e400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11862,199_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op5,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op5,kernel_graph0,{12307-12309},12309,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12298,12332,0x12c20320ea00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11862,200_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op3,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op3,kernel_graph0,{12328-12330},12330,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12301,12359,0x12c23960fe00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11862,201_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op3,kernel_graph0,{12305-12350-12357},12357,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12302,12305,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11862,201_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op3,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12307,12317,0x12c24ee10200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11864,203_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op4,kernel_graph0,{12315},12315,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12309,12314,0x12c256e10400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11865,204_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op5,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op5,kernel_graph0,{12312},12312,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12312,12318,0x12c1f920e400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11865,205_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op1,kernel_graph0,{12315},12315,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12315,12321,0x12c256e10400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11865,206_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op1,kernel_graph0,{12319},12319,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12319,12325,0x12c24ee10200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11866,207_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op6,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op6,kernel_graph0,{12323},12323,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12322,12373,0x12c1f920e400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11867,208_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op2,kernel_graph0,{12326-12364-12371},12371,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12323,12326,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11867,208_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12328,12338,0x12c24ee10200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11868,210_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op2,kernel_graph0,{12336},12336,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12330,12335,0x12c256e10400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11870,211_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op3,kernel_graph0,{12333},12333,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12333,12339,0x12c20320ea00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11871,212_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op0,kernel_graph0,{12336},12336,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12336,12342,0x12c256e10400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11871,213_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op0,kernel_graph0,{12340},12340,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12340,12346,0x12c1fd20e600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11872,214_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op4,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op4,kernel_graph0,{12344},12344,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12343,12388,0x12c20320ea00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11873,215_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op1,kernel_graph0,{12347-12378-12386},12386,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12344,12347,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11873,215_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12349,12355,0x12c241610200,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11875,217_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op24,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op24,kernel_graph0,{12353},12353,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12350,12352,0x12c1ae00d400,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11875,217_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op24,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op24,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12353,12706,0x12c1fd20e600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11875,218_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op24,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op24,kernel_graph0,{12704},12704,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12356,12396,0x12c20720ec00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11877,219_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op7,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op7,kernel_graph0,{12361-12392},12392,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12357,12361,0x12c1ae00d400,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11877,219_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op7,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op7,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12363,12369,0x12c20120e800,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11878,221_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op23,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op23,kernel_graph0,{12367},12367,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12364,12366,0x12c1ae00d400,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11878,221_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op23,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op23,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12367,12703,0x12c23960fe00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11881,222_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op23,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op23,kernel_graph0,{12701},12701,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12370,12395,0x12c24ee10200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11882,223_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op6,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op6,kernel_graph0,{12375-12392},12392,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12371,12375,0x12c1ae00d400,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11882,223_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op6,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op6,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12377,12384,0x12c20120e800,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11885,225_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op22,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op22,kernel_graph0,{12382},12382,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12378,12381,0x12c19400c800,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11885,225_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op22,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op22,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12382,12700,0x12c1ef20de00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11886,226_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op22,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op22,kernel_graph0,{12698},12698,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12385,12394,0x12c1f920e400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11886,227_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op5,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op5,kernel_graph0,{12390-12392},12392,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12386,12390,0x12c19400c800,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11886,227_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op5,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op5,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12392,12399,0x12c20120e800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11888,229_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/AddN-op0,kernel_graph0,{12397},12397,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12397,12407,0x12c24ee10200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11889,230_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op7,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op7,kernel_graph0,{12405},12405,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12400,12408,0x12c256e10400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11890,231_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op2,kernel_graph0,{12405},12405,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12403,12413,0x12c1f320e000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11890,232_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op2,kernel_graph0,{12411},12411,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12404,12733,0x12c187931600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11890,232_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op2,kernel_graph0,{12731},12731,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12405,12410,0x12c25ee10600,0,DefaultEnhancedAscendMemoryPool,16778752,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11890,232_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12411,12416,0x12c20120e800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11892,233_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op2,kernel_graph0,{12414},12414,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12414,12499,0x12c20520ea00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11892,234_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/AddN-op0,kernel_graph0,{12417-12420-12428-12496},12496,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12419,12426,0x12c23d610000,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11893,236_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op20,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op20,kernel_graph0,{12424},12424,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12420,12423,0x12c242c10200,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11893,236_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op20,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op20,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12424,12694,0x12c1df00da00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11894,237_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op20,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op20,kernel_graph0,{12692},12692,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12427,12440,0x12c24ee10200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11895,238_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op8,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op8,kernel_graph0,{12431-12433-12437},12437,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12428,12431,0x12c19400c800,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11895,238_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op8,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op8,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12433,12456,0x12c20f20f000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11898,240_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op6,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op6,kernel_graph0,{12435-12447-12454},12454,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12437,12444,0x12c1d440d800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11901,242_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op7,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op7,kernel_graph0,{12440-12442},12442,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12442,12471,0x12c1b960d200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11902,244_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/Grad_SiLU/SiLUGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/Grad_SiLU/SiLUGrad-op1,kernel_graph0,{12461-12469},12469,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12446,12452,0x12c1e9c0dc00,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11904,245_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op21,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op21,kernel_graph0,{12450},12450,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12447,12449,0x12c19400c800,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11904,245_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op21,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op21,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12450,12697,0x12c24ee10200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11905,246_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op21,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op21,kernel_graph0,{12695},12695,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12453,12477,0x12c20120e800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11905,247_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op9,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op9,kernel_graph0,{12458-12475},12475,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12454,12458,0x12c19400c800,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11905,247_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op9,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op9,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12460,12467,0x12c1b400d000,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11908,249_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op19,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op19,kernel_graph0,{12465},12465,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12461,12464,0x12c19400c800,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11908,249_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op19,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op19,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12465,12691,0x12c1c980d600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11908,250_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op19,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op19,kernel_graph0,{12689},12689,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12468,12478,0x12c1a400ce00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11909,251_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op10,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op10,kernel_graph0,{12473-12475},12475,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12469,12473,0x12c19400c800,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11909,251_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op10,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op10,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12475,12481,0x12c1e9c0dc00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11910,253_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/AddN-op0,kernel_graph0,{12479},12479,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12479,12489,0x12c1f320e000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11912,254_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op8,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op8,kernel_graph0,{12487},12487,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12482,12490,0x12c1d440d800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11913,255_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op2,kernel_graph0,{12487},12487,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12485,12495,0x12c1a400ce00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11913,256_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op3,kernel_graph0,{12493},12493,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12486,12724,0x12c187929400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11913,256_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op3,kernel_graph0,{12722},12722,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12487,12492,0x12c19400c800,0,DefaultEnhancedAscendMemoryPool,16778752,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11913,256_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op3,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12493,12498,0x12c20120e800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11916,257_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op3,kernel_graph0,{12496},12496,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12496,12661,0x12c1e9c0dc00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11917,258_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/AddN-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/AddN-op1,kernel_graph0,{12499-12502-12510-12658},12658,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12501,12508,0x12c259a10400,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11919,260_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op18,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op18,kernel_graph0,{12506},12506,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12502,12505,0x12c1edc0de00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11919,260_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op18,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op18,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12506,12688,0x12c25ba10600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11920,261_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op18,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op18,kernel_graph0,{12686},12686,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12509,12518,0x12c23d610000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11920,262_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op11,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op11,kernel_graph0,{12513-12516},12516,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12510,12513,0x12c1edc0de00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11920,262_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op11,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op11,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12515,12529,0x12c1f320e000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11922,264_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op4,kernel_graph0,{12524},12524,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12516,12519,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11922,264_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op4,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12520,12542,0x12c1f720e200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11922,265_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,kernel_graph0,{12540},12540,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12521,12539,0x12c23d610000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11922,265_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,kernel_graph0,{12537},12537,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12522,12546,0x12c20120e800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11922,265_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,kernel_graph0,{12544},12544,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12523,12535,0x12c180008800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11922,265_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12524,12536,0x12c20f20f000,0,DefaultEnhancedAscendMemoryPool,515969024,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11922,265_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12537,12553,0x12c18e00c400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11924,266_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op11,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op11,kernel_graph0,{12549-12551},12551,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12540,12576,0x12c1d440d800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11924,267_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op9,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op9,kernel_graph0,{12570-12573},12573,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12543,12603,0x12c20520ea00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11925,268_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op7,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op7,kernel_graph0,{12547-12594-12601},12601,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12544,12547,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11925,268_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op7,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op7,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12549,12559,0x12c1f320e000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11927,270_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op10,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op10,kernel_graph0,{12557},12557,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12551,12556,0x12c20f20f000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11928,271_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op11,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op11,kernel_graph0,{12554},12554,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12554,12560,0x12c18e00c400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11928,272_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op3,kernel_graph0,{12557},12557,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12557,12563,0x12c20f20f000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11931,273_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op1,kernel_graph0,{12561},12561,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12561,12567,0x12c20120e800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11932,274_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op12,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op12,kernel_graph0,{12565},12565,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12564,12617,0x12c23d610000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11932,275_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op6,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op6,kernel_graph0,{12568-12608-12615},12615,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12565,12568,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11932,275_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op6,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op6,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12570,12582,0x12c18e00c400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11935,277_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op8,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op8,kernel_graph0,{12580},12580,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12573,12579,0x12c1f320e000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11935,278_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op9,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op9,kernel_graph0,{12577},12577,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12577,12583,0x12c1d440d800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11936,279_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op2,kernel_graph0,{12580},12580,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12580,12586,0x12c1f320e000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11936,280_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op0,kernel_graph0,{12584},12584,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12584,12590,0x12c20120e800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11937,281_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op10,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op10,kernel_graph0,{12588},12588,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12587,12632,0x12c18e00c400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11937,282_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op5,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op5,kernel_graph0,{12591-12622-12630},12630,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12588,12591,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11937,282_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op5,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op5,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12593,12599,0x12c20920ec00,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11940,284_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op17,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op17,kernel_graph0,{12597},12597,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12594,12596,0x12c1edc0de00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11940,284_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op17,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op17,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12597,12685,0x12c20120e800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11940,285_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op17,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op17,kernel_graph0,{12683},12683,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12600,12640,0x12c1f320e000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11942,286_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op14,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op14,kernel_graph0,{12605-12636},12636,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12601,12605,0x12c1edc0de00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11942,286_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op14,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op14,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12607,12613,0x12c259a10400,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11943,288_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op16,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op16,kernel_graph0,{12611},12611,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12608,12610,0x12c1edc0de00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11943,288_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op16,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op16,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12611,12682,0x12c1f720e200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11946,289_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op16,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op16,kernel_graph0,{12680},12680,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12614,12639,0x12c20520ea00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11947,290_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op13,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op13,kernel_graph0,{12619-12636},12636,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12615,12619,0x12c1edc0de00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11947,290_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op13,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op13,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12621,12628,0x12c1fb20e400,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11949,292_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op15,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op15,kernel_graph0,{12626},12626,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12622,12625,0x12c1edc0de00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11949,292_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op15,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op15,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12626,12679,0x12c23d610000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11949,293_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op15,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op15,kernel_graph0,{12677},12677,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12629,12638,0x12c1800a0600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11950,294_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op12,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op12,kernel_graph0,{12634-12636},12636,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12630,12634,0x12c1edc0de00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11950,294_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op12,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op12,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12636,12643,0x12c1d440d800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11952,296_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/AddN-op0,kernel_graph0,{12641},12641,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12641,12651,0x12c20f20f000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11953,297_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op13,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op13,kernel_graph0,{12649},12649,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12644,12652,0x12c1d440d800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11953,298_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op2,kernel_graph0,{12649},12649,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12647,12657,0x12c21720f200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11954,299_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op4,kernel_graph0,{12655},12655,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12648,12727,0x12c180014200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11954,299_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op4,kernel_graph0,{12725},12725,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12649,12654,0x12c1edc0de00,0,DefaultEnhancedAscendMemoryPool,16778752,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11954,299_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op4,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12655,12660,0x12c1f320e000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11955,300_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op4,kernel_graph0,{12658},12658,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12658,12664,0x12c20520ea00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11957,301_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/AddN-op0,kernel_graph0,{12662},12662,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12662,12667,0x12c1d440d800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11957,302_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op14,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op14,kernel_graph0,{12665},12665,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12665,12676,0x12c20f20f000,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11958,303_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/tok_embeddings-LlamaEmbedding/Grad_Gather/UnsortedSegmentSum-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/tok_embeddings-LlamaEmbedding/Grad_Gather/UnsortedSegmentSum-op0,kernel_graph0,{12674},12674,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12669,12673,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11958,304_Default/Mul-op1,Default/Mul-op1,kernel_graph0,{12671},12671,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12671,12737,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11965,305_Default/Reciprocal-op0,Default/Reciprocal-op0,kernel_graph0,{12674-12677-12680-12683-12686-12689-12692-12695-12698-12701-12704-12707-12710-12713-12716-12719-12722-12725-12728-12731-12734},12734,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12674,12972,0x12c187935800,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11965,306_Default/Mul-op2,Default/Mul-op2,kernel_graph0,{12738-12898-12970},12970,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12677,12975,0x12c1f320e000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11966,307_Default/Mul-op3,Default/Mul-op3,kernel_graph0,{12746-12898-12973},12973,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12680,12978,0x12c1e9c0dc00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11966,308_Default/Mul-op4,Default/Mul-op4,kernel_graph0,{12754-12898-12976},12976,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12683,12981,0x12c1f720e200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11966,309_Default/Mul-op5,Default/Mul-op5,kernel_graph0,{12762-12898-12979},12979,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12686,12984,0x12c23d610000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11967,310_Default/Mul-op6,Default/Mul-op6,kernel_graph0,{12770-12898-12982},12982,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12689,12987,0x12c1d440d800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11969,311_Default/Mul-op7,Default/Mul-op7,kernel_graph0,{12778-12898-12985},12985,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12692,12990,0x12c20f20f000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11969,312_Default/Mul-op8,Default/Mul-op8,kernel_graph0,{12786-12898-12988},12988,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12695,12993,0x12c1df00da00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11970,313_Default/Mul-op9,Default/Mul-op9,kernel_graph0,{12794-12898-12991},12991,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12698,12996,0x12c180018400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11970,314_Default/Mul-op10,Default/Mul-op10,kernel_graph0,{12802-12898-12994},12994,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12701,12999,0x12c1edc0de00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11971,315_Default/Mul-op11,Default/Mul-op11,kernel_graph0,{12810-12898-12997},12997,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12704,13002,0x12c23960fe00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11971,316_Default/Mul-op12,Default/Mul-op12,kernel_graph0,{12818-12898-13000},13000,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12707,13005,0x12c1fb20e400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11972,317_Default/Mul-op13,Default/Mul-op13,kernel_graph0,{12826-12898-13003},13003,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12710,13008,0x12c1ff20e600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11973,318_Default/Mul-op14,Default/Mul-op14,kernel_graph0,{12834-12898-13006},13006,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12713,13011,0x12c219e0f200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11974,319_Default/Mul-op15,Default/Mul-op15,kernel_graph0,{12842-12898-13009},13009,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12716,13014,0x12c241610200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11974,320_Default/Mul-op16,Default/Mul-op16,kernel_graph0,{12850-12898-13012},13012,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12719,13017,0x12c24c210400,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11976,321_Default/Mul-op17,Default/Mul-op17,kernel_graph0,{12858-12898-13015},13015,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12722,13020,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11977,322_Default/Mul-op18,Default/Mul-op18,kernel_graph0,{12866-12898-13018},13018,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12725,13023,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11977,323_Default/Mul-op19,Default/Mul-op19,kernel_graph0,{12874-12898-13021},13021,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12728,13026,0x12c180014200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11978,324_Default/Mul-op20,Default/Mul-op20,kernel_graph0,{12882-12898-13024},13024,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12731,13029,0x12c180010000,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11979,325_Default/Mul-op21,Default/Mul-op21,kernel_graph0,{12890-12898-13027},13027,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12734,13032,0x12c1f1c0e000,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11980,326_Default/Mul-op22,Default/Mul-op22,kernel_graph0,{12898-12900-13030},13030,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12738,12743,0x12c1a6d35a00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11981,327_Default/clip_grad_norm-ClipGradNorm/Square-op0,Default/clip_grad_norm-ClipGradNorm/Square-op0,kernel_graph0,{12741},12741,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12740,12910,0x12c180008a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11981,328_Default/clip_grad_norm-ClipGradNorm/SumExt-op0,Default/clip_grad_norm-ClipGradNorm/SumExt-op0,kernel_graph0,{12744-12908},12908,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12741,12744,0x12c18000be00,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11981,328_Default/clip_grad_norm-ClipGradNorm/SumExt-op0,Default/clip_grad_norm-ClipGradNorm/SumExt-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12746,12751,0x12c209e0e800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11983,330_Default/clip_grad_norm-ClipGradNorm/Square-op1,Default/clip_grad_norm-ClipGradNorm/Square-op1,kernel_graph0,{12749},12749,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12748,12911,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11984,331_Default/clip_grad_norm-ClipGradNorm/SumExt-op1,Default/clip_grad_norm-ClipGradNorm/SumExt-op1,kernel_graph0,{12752-12908},12908,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12749,12752,0x12c18000be00,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11984,331_Default/clip_grad_norm-ClipGradNorm/SumExt-op1,Default/clip_grad_norm-ClipGradNorm/SumExt-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12754,12759,0x12c209e0e800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11987,333_Default/clip_grad_norm-ClipGradNorm/Square-op2,Default/clip_grad_norm-ClipGradNorm/Square-op2,kernel_graph0,{12757},12757,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12756,12912,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11988,334_Default/clip_grad_norm-ClipGradNorm/SumExt-op2,Default/clip_grad_norm-ClipGradNorm/SumExt-op2,kernel_graph0,{12760-12908},12908,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12757,12760,0x12c18000be00,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11988,334_Default/clip_grad_norm-ClipGradNorm/SumExt-op2,Default/clip_grad_norm-ClipGradNorm/SumExt-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12762,12767,0x12c209e0e800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11991,336_Default/clip_grad_norm-ClipGradNorm/Square-op3,Default/clip_grad_norm-ClipGradNorm/Square-op3,kernel_graph0,{12765},12765,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12764,12913,0x12c180009a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11992,337_Default/clip_grad_norm-ClipGradNorm/SumExt-op3,Default/clip_grad_norm-ClipGradNorm/SumExt-op3,kernel_graph0,{12768-12908},12908,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12765,12768,0x12c18000be00,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11992,337_Default/clip_grad_norm-ClipGradNorm/SumExt-op3,Default/clip_grad_norm-ClipGradNorm/SumExt-op3,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12770,12775,0x12c209e0e800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11993,339_Default/clip_grad_norm-ClipGradNorm/Square-op4,Default/clip_grad_norm-ClipGradNorm/Square-op4,kernel_graph0,{12773},12773,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12772,12914,0x12c180009c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11994,340_Default/clip_grad_norm-ClipGradNorm/SumExt-op4,Default/clip_grad_norm-ClipGradNorm/SumExt-op4,kernel_graph0,{12776-12908},12908,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12773,12776,0x12c18000be00,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11994,340_Default/clip_grad_norm-ClipGradNorm/SumExt-op4,Default/clip_grad_norm-ClipGradNorm/SumExt-op4,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12778,12783,0x12c224a0f400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11996,342_Default/clip_grad_norm-ClipGradNorm/Square-op5,Default/clip_grad_norm-ClipGradNorm/Square-op5,kernel_graph0,{12781},12781,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12780,12915,0x12c180009e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11997,343_Default/clip_grad_norm-ClipGradNorm/SumExt-op5,Default/clip_grad_norm-ClipGradNorm/SumExt-op5,kernel_graph0,{12784-12908},12908,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12781,12784,0x12c18000be00,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11997,343_Default/clip_grad_norm-ClipGradNorm/SumExt-op5,Default/clip_grad_norm-ClipGradNorm/SumExt-op5,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12786,12791,0x12c224a0f400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11999,345_Default/clip_grad_norm-ClipGradNorm/Square-op6,Default/clip_grad_norm-ClipGradNorm/Square-op6,kernel_graph0,{12789},12789,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12788,12916,0x12c18000a000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11999,346_Default/clip_grad_norm-ClipGradNorm/SumExt-op6,Default/clip_grad_norm-ClipGradNorm/SumExt-op6,kernel_graph0,{12792-12908},12908,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12789,12792,0x12c18000be00,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,11999,346_Default/clip_grad_norm-ClipGradNorm/SumExt-op6,Default/clip_grad_norm-ClipGradNorm/SumExt-op6,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12794,12799,0x12c224a0f400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12001,348_Default/clip_grad_norm-ClipGradNorm/Square-op7,Default/clip_grad_norm-ClipGradNorm/Square-op7,kernel_graph0,{12797},12797,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12796,12917,0x12c18000a200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12002,349_Default/clip_grad_norm-ClipGradNorm/SumExt-op7,Default/clip_grad_norm-ClipGradNorm/SumExt-op7,kernel_graph0,{12800-12908},12908,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12797,12800,0x12c18000be00,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12002,349_Default/clip_grad_norm-ClipGradNorm/SumExt-op7,Default/clip_grad_norm-ClipGradNorm/SumExt-op7,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12802,12807,0x12c209e0e800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12003,351_Default/clip_grad_norm-ClipGradNorm/Square-op8,Default/clip_grad_norm-ClipGradNorm/Square-op8,kernel_graph0,{12805},12805,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12804,12918,0x12c18000a400,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12005,352_Default/clip_grad_norm-ClipGradNorm/SumExt-op8,Default/clip_grad_norm-ClipGradNorm/SumExt-op8,kernel_graph0,{12808-12908},12908,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12805,12808,0x12c18000be00,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12005,352_Default/clip_grad_norm-ClipGradNorm/SumExt-op8,Default/clip_grad_norm-ClipGradNorm/SumExt-op8,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12810,12815,0x12c209e0e800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12012,354_Default/clip_grad_norm-ClipGradNorm/Square-op9,Default/clip_grad_norm-ClipGradNorm/Square-op9,kernel_graph0,{12813},12813,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12812,12919,0x12c18000a600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12012,355_Default/clip_grad_norm-ClipGradNorm/SumExt-op9,Default/clip_grad_norm-ClipGradNorm/SumExt-op9,kernel_graph0,{12816-12908},12908,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12813,12816,0x12c18000be00,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12012,355_Default/clip_grad_norm-ClipGradNorm/SumExt-op9,Default/clip_grad_norm-ClipGradNorm/SumExt-op9,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12818,12823,0x12c209e0e800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12014,357_Default/clip_grad_norm-ClipGradNorm/Square-op10,Default/clip_grad_norm-ClipGradNorm/Square-op10,kernel_graph0,{12821},12821,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12820,12920,0x12c18000a800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12015,358_Default/clip_grad_norm-ClipGradNorm/SumExt-op10,Default/clip_grad_norm-ClipGradNorm/SumExt-op10,kernel_graph0,{12824-12908},12908,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12821,12824,0x12c18000be00,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12015,358_Default/clip_grad_norm-ClipGradNorm/SumExt-op10,Default/clip_grad_norm-ClipGradNorm/SumExt-op10,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12826,12831,0x12c209e0e800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12017,360_Default/clip_grad_norm-ClipGradNorm/Square-op11,Default/clip_grad_norm-ClipGradNorm/Square-op11,kernel_graph0,{12829},12829,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12828,12921,0x12c18000aa00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12018,361_Default/clip_grad_norm-ClipGradNorm/SumExt-op11,Default/clip_grad_norm-ClipGradNorm/SumExt-op11,kernel_graph0,{12832-12908},12908,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12829,12832,0x12c18000be00,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12018,361_Default/clip_grad_norm-ClipGradNorm/SumExt-op11,Default/clip_grad_norm-ClipGradNorm/SumExt-op11,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12834,12839,0x12c224a0f400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12019,363_Default/clip_grad_norm-ClipGradNorm/Square-op12,Default/clip_grad_norm-ClipGradNorm/Square-op12,kernel_graph0,{12837},12837,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12836,12922,0x12c18000ac00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12019,364_Default/clip_grad_norm-ClipGradNorm/SumExt-op12,Default/clip_grad_norm-ClipGradNorm/SumExt-op12,kernel_graph0,{12840-12908},12908,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12837,12840,0x12c18000be00,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12019,364_Default/clip_grad_norm-ClipGradNorm/SumExt-op12,Default/clip_grad_norm-ClipGradNorm/SumExt-op12,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12842,12847,0x12c224a0f400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12021,366_Default/clip_grad_norm-ClipGradNorm/Square-op13,Default/clip_grad_norm-ClipGradNorm/Square-op13,kernel_graph0,{12845},12845,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12844,12923,0x12c18000ae00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12022,367_Default/clip_grad_norm-ClipGradNorm/SumExt-op13,Default/clip_grad_norm-ClipGradNorm/SumExt-op13,kernel_graph0,{12848-12908},12908,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12845,12848,0x12c18000be00,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12022,367_Default/clip_grad_norm-ClipGradNorm/SumExt-op13,Default/clip_grad_norm-ClipGradNorm/SumExt-op13,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12850,12855,0x12c224a0f400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12024,369_Default/clip_grad_norm-ClipGradNorm/Square-op14,Default/clip_grad_norm-ClipGradNorm/Square-op14,kernel_graph0,{12853},12853,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12852,12924,0x12c18000b000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12025,370_Default/clip_grad_norm-ClipGradNorm/SumExt-op14,Default/clip_grad_norm-ClipGradNorm/SumExt-op14,kernel_graph0,{12856-12908},12908,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12853,12856,0x12c18000be00,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12025,370_Default/clip_grad_norm-ClipGradNorm/SumExt-op14,Default/clip_grad_norm-ClipGradNorm/SumExt-op14,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12858,12863,0x12c1a6d35a00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12028,372_Default/clip_grad_norm-ClipGradNorm/Square-op15,Default/clip_grad_norm-ClipGradNorm/Square-op15,kernel_graph0,{12861},12861,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12860,12925,0x12c18000b200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12029,373_Default/clip_grad_norm-ClipGradNorm/SumExt-op15,Default/clip_grad_norm-ClipGradNorm/SumExt-op15,kernel_graph0,{12864-12908},12908,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12861,12864,0x12c18000be00,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12029,373_Default/clip_grad_norm-ClipGradNorm/SumExt-op15,Default/clip_grad_norm-ClipGradNorm/SumExt-op15,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12866,12871,0x12c18000be00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12032,375_Default/clip_grad_norm-ClipGradNorm/Square-op16,Default/clip_grad_norm-ClipGradNorm/Square-op16,kernel_graph0,{12869},12869,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12868,12926,0x12c18000b400,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12033,376_Default/clip_grad_norm-ClipGradNorm/SumExt-op16,Default/clip_grad_norm-ClipGradNorm/SumExt-op16,kernel_graph0,{12872-12908},12908,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12869,12872,0x12c1f1c12200,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12033,376_Default/clip_grad_norm-ClipGradNorm/SumExt-op16,Default/clip_grad_norm-ClipGradNorm/SumExt-op16,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12874,12879,0x12c18000be00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12034,378_Default/clip_grad_norm-ClipGradNorm/Square-op17,Default/clip_grad_norm-ClipGradNorm/Square-op17,kernel_graph0,{12877},12877,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12876,12927,0x12c18000b600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12035,379_Default/clip_grad_norm-ClipGradNorm/SumExt-op17,Default/clip_grad_norm-ClipGradNorm/SumExt-op17,kernel_graph0,{12880-12908},12908,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12877,12880,0x12c1f1c12200,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12035,379_Default/clip_grad_norm-ClipGradNorm/SumExt-op17,Default/clip_grad_norm-ClipGradNorm/SumExt-op17,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12882,12887,0x12c18000be00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12037,381_Default/clip_grad_norm-ClipGradNorm/Square-op18,Default/clip_grad_norm-ClipGradNorm/Square-op18,kernel_graph0,{12885},12885,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12884,12928,0x12c18000b800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12038,382_Default/clip_grad_norm-ClipGradNorm/SumExt-op18,Default/clip_grad_norm-ClipGradNorm/SumExt-op18,kernel_graph0,{12888-12908},12908,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12885,12888,0x12c1f1c12200,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12038,382_Default/clip_grad_norm-ClipGradNorm/SumExt-op18,Default/clip_grad_norm-ClipGradNorm/SumExt-op18,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12890,12895,0x12c18000be00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12040,384_Default/clip_grad_norm-ClipGradNorm/Square-op19,Default/clip_grad_norm-ClipGradNorm/Square-op19,kernel_graph0,{12893},12893,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12892,12929,0x12c1f1c12200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12040,385_Default/clip_grad_norm-ClipGradNorm/SumExt-op19,Default/clip_grad_norm-ClipGradNorm/SumExt-op19,kernel_graph0,{12896-12908},12908,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12893,12896,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12040,385_Default/clip_grad_norm-ClipGradNorm/SumExt-op19,Default/clip_grad_norm-ClipGradNorm/SumExt-op19,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12898,16263,0x12c18000be00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12042,387_Default/AllFinite-op0,Default/AllFinite-op0,kernel_graph0,{12934-12937-12939},12939,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12900,12905,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12043,388_Default/clip_grad_norm-ClipGradNorm/Square-op20,Default/clip_grad_norm-ClipGradNorm/Square-op20,kernel_graph0,{12903},12903,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12902,12930,0x12c18000c000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12044,389_Default/clip_grad_norm-ClipGradNorm/SumExt-op20,Default/clip_grad_norm-ClipGradNorm/SumExt-op20,kernel_graph0,{12906-12908},12908,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12903,12906,0x12c18000c200,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12044,389_Default/clip_grad_norm-ClipGradNorm/SumExt-op20,Default/clip_grad_norm-ClipGradNorm/SumExt-op20,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12908,12933,0x12c18000c200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12047,391_Default/clip_grad_norm-ClipGradNorm/AddN-op0,Default/clip_grad_norm-ClipGradNorm/AddN-op0,kernel_graph0,{12931},12931,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12931,16260,0x12c180008a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12047,392_Default/clip_grad_norm-ClipGradNorm/Sqrt-op0,Default/clip_grad_norm-ClipGradNorm/Sqrt-op0,kernel_graph0,{12941-12943},12943,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12934,12964,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12049,393_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Select-op1,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Select-op1,kernel_graph0,{12955-12960},12960,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12937,13081,0x12c180008c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12050,394_Default/LogicalNot-op0,Default/LogicalNot-op0,kernel_graph0,{12957},12957,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12939,12967,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12050,395_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/LogicalOr-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/LogicalOr-op0,kernel_graph0,{12965},12965,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12941,12945,0x12c180009a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12053,396_Default/clip_grad_norm-ClipGradNorm/GreaterEqual-op0,Default/clip_grad_norm-ClipGradNorm/GreaterEqual-op0,kernel_graph0,{12943},12943,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12943,12948,0x12c180009c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12054,397_Default/clip_grad_norm-ClipGradNorm/Select-op0,Default/clip_grad_norm-ClipGradNorm/Select-op0,kernel_graph0,{12946},12946,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12946,12951,0x12c180009a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12054,398_Default/learning_rate-CosineWithWarmUpLR/Add-op0,Default/learning_rate-CosineWithWarmUpLR/Add-op0,kernel_graph0,{12949},12949,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12949,12954,0x12c180009c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12055,399_Default/clip_grad_norm-ClipGradNorm/Div-op0,Default/clip_grad_norm-ClipGradNorm/Div-op0,kernel_graph0,{12952},12952,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12952,13033,0x12c180009a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12056,400_Default/clip_grad_norm-ClipGradNorm/Minimum-op0,Default/clip_grad_norm-ClipGradNorm/Minimum-op0,kernel_graph0,{12970-12973-12976-12979-12982-12985-12988-12991-12994-12997-13000-13003-13006-13009-13012-13015-13018-13021-13024-13027-13030},13030,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12955,12963,0x12c180009c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12056,401_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Mul-op1,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Mul-op1,kernel_graph0,{12960},12960,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12957,12962,0x12c180009e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12057,402_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/LogicalAnd-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/LogicalAnd-op0,kernel_graph0,{12960},12960,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12960,13039,0x12c180009600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12057,403_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Select-op2,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Select-op2,kernel_graph0,{13037},13037,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12965,13035,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12058,404_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Select-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Select-op0,kernel_graph0,{13033},13033,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12968,13037,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12058,405_Default/learning_rate-CosineWithWarmUpLR/Add-op1,Default/learning_rate-CosineWithWarmUpLR/Add-op1,kernel_graph0,{13035},13035,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12970,13240,0x12c1a6d35a00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12058,406_Default/clip_grad_norm-ClipGradNorm/Mul-op0,Default/clip_grad_norm-ClipGradNorm/Mul-op0,kernel_graph0,{13236-13238},13238,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12973,13230,0x12c209e0e800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12059,407_Default/clip_grad_norm-ClipGradNorm/Mul-op1,Default/clip_grad_norm-ClipGradNorm/Mul-op1,kernel_graph0,{13226-13228},13228,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12976,13222,0x12c1f1c12200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12059,408_Default/clip_grad_norm-ClipGradNorm/Mul-op2,Default/clip_grad_norm-ClipGradNorm/Mul-op2,kernel_graph0,{13218-13220},13220,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12979,13214,0x12c1e9c0dc00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12060,409_Default/clip_grad_norm-ClipGradNorm/Mul-op3,Default/clip_grad_norm-ClipGradNorm/Mul-op3,kernel_graph0,{13210-13212},13212,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12982,13206,0x12c1f5c12400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12060,410_Default/clip_grad_norm-ClipGradNorm/Mul-op4,Default/clip_grad_norm-ClipGradNorm/Mul-op4,kernel_graph0,{13202-13204},13204,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12985,13198,0x12c1c6135c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12064,411_Default/clip_grad_norm-ClipGradNorm/Mul-op5,Default/clip_grad_norm-ClipGradNorm/Mul-op5,kernel_graph0,{13194-13196},13196,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12988,13190,0x12c1d0d35e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12064,412_Default/clip_grad_norm-ClipGradNorm/Mul-op6,Default/clip_grad_norm-ClipGradNorm/Mul-op6,kernel_graph0,{13186-13188},13188,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12991,13182,0x12c20de0ea00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12065,413_Default/clip_grad_norm-ClipGradNorm/Mul-op7,Default/clip_grad_norm-ClipGradNorm/Mul-op7,kernel_graph0,{13178-13180},13180,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12994,13174,0x12c23d610000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12065,414_Default/clip_grad_norm-ClipGradNorm/Mul-op8,Default/clip_grad_norm-ClipGradNorm/Mul-op8,kernel_graph0,{13170-13172},13172,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 12997,13166,0x12c1db936000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12065,415_Default/clip_grad_norm-ClipGradNorm/Mul-op9,Default/clip_grad_norm-ClipGradNorm/Mul-op9,kernel_graph0,{13162-13164},13164,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13000,13158,0x12c1edc0de00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12065,416_Default/clip_grad_norm-ClipGradNorm/Mul-op10,Default/clip_grad_norm-ClipGradNorm/Mul-op10,kernel_graph0,{13154-13156},13156,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13003,13150,0x12c1df936200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12068,417_Default/clip_grad_norm-ClipGradNorm/Mul-op11,Default/clip_grad_norm-ClipGradNorm/Mul-op11,kernel_graph0,{13146-13148},13148,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13006,13142,0x12c224a0f400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12069,418_Default/clip_grad_norm-ClipGradNorm/Mul-op12,Default/clip_grad_norm-ClipGradNorm/Mul-op12,kernel_graph0,{13138-13140},13140,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13009,13134,0x12c22f60f600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12069,419_Default/clip_grad_norm-ClipGradNorm/Mul-op13,Default/clip_grad_norm-ClipGradNorm/Mul-op13,kernel_graph0,{13130-13132},13132,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13012,13126,0x12c218a0ec00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12071,420_Default/clip_grad_norm-ClipGradNorm/Mul-op14,Default/clip_grad_norm-ClipGradNorm/Mul-op14,kernel_graph0,{13122-13124},13124,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13015,13118,0x12c180018400,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12071,421_Default/clip_grad_norm-ClipGradNorm/Mul-op15,Default/clip_grad_norm-ClipGradNorm/Mul-op15,kernel_graph0,{13114-13116},13116,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13018,13110,0x12c22360ee00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12072,422_Default/clip_grad_norm-ClipGradNorm/Mul-op16,Default/clip_grad_norm-ClipGradNorm/Mul-op16,kernel_graph0,{13106-13108},13108,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13021,13102,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12073,423_Default/clip_grad_norm-ClipGradNorm/Mul-op17,Default/clip_grad_norm-ClipGradNorm/Mul-op17,kernel_graph0,{13098-13100},13100,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13024,13094,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12074,424_Default/clip_grad_norm-ClipGradNorm/Mul-op18,Default/clip_grad_norm-ClipGradNorm/Mul-op18,kernel_graph0,{13090-13092},13092,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13027,13086,0x12c180014200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12074,425_Default/clip_grad_norm-ClipGradNorm/Mul-op19,Default/clip_grad_norm-ClipGradNorm/Mul-op19,kernel_graph0,{13082-13084},13084,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13030,13248,0x12c18000c000,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12075,426_Default/clip_grad_norm-ClipGradNorm/Mul-op20,Default/clip_grad_norm-ClipGradNorm/Mul-op20,kernel_graph0,{13244-13246},13246,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13040,13049,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12078,430_Default/learning_rate-CosineWithWarmUpLR/Cast-op0,Default/learning_rate-CosineWithWarmUpLR/Cast-op0,kernel_graph0,{13042-13045-13047},13047,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13042,13044,0x12c180009600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,12079,431_Default/learning_rate-CosineWithWarmUpLR/GreaterEqual-op0,Default/learning_rate-CosineWithWarmUpLR/GreaterEqual-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13045,13050,0x12c180009600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13044,435_Default/learning_rate-CosineWithWarmUpLR/Greater-op0,Default/learning_rate-CosineWithWarmUpLR/Greater-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13047,13053,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13045,436_Default/TensorMove-op0,Default/TensorMove-op0,kernel_graph0,{13051},13051,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13051,13056,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13050,440_Default/learning_rate-CosineWithWarmUpLR/Sub-op1,Default/learning_rate-CosineWithWarmUpLR/Sub-op1,kernel_graph0,{13054},13054,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13054,13059,0x12c180009600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13051,441_Default/learning_rate-CosineWithWarmUpLR/Div-op1,Default/learning_rate-CosineWithWarmUpLR/Div-op1,kernel_graph0,{13057},13057,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13057,13062,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13052,442_Default/learning_rate-CosineWithWarmUpLR/Mul-op1,Default/learning_rate-CosineWithWarmUpLR/Mul-op1,kernel_graph0,{13060},13060,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13060,13065,0x12c180009600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13052,443_Default/learning_rate-CosineWithWarmUpLR/Cos-op0,Default/learning_rate-CosineWithWarmUpLR/Cos-op0,kernel_graph0,{13063},13063,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13063,13068,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13054,444_Default/learning_rate-CosineWithWarmUpLR/Add-op3,Default/learning_rate-CosineWithWarmUpLR/Add-op3,kernel_graph0,{13066},13066,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13066,13071,0x12c180009600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13055,445_Default/learning_rate-CosineWithWarmUpLR/Mul-op2,Default/learning_rate-CosineWithWarmUpLR/Mul-op2,kernel_graph0,{13069},13069,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13069,13074,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13055,446_Default/learning_rate-CosineWithWarmUpLR/Maximum-op0,Default/learning_rate-CosineWithWarmUpLR/Maximum-op0,kernel_graph0,{13072},13072,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13072,13077,0x12c180009600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13056,447_Default/learning_rate-CosineWithWarmUpLR/Mul-op3,Default/learning_rate-CosineWithWarmUpLR/Mul-op3,kernel_graph0,{13075},13075,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13075,13080,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13057,448_Default/learning_rate-CosineWithWarmUpLR/Add-op4,Default/learning_rate-CosineWithWarmUpLR/Add-op4,kernel_graph0,{13078},13078,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13078,16261,0x12c180009600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13060,451_Default/Reshape-op0,Default/Reshape-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13082,13911,0x12c1f1c0e000,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13081,453_Default/optimizer-FP32StateAdamWeightDecay/Mul-op96,Default/optimizer-FP32StateAdamWeightDecay/Mul-op96,kernel_graph0,{13908},13908,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13084,13089,0x12c223613000,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13082,454_Default/optimizer-FP32StateAdamWeightDecay/Square-op19,Default/optimizer-FP32StateAdamWeightDecay/Square-op19,kernel_graph0,{13087},13087,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13087,13907,0x12c180010200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13083,455_Default/optimizer-FP32StateAdamWeightDecay/Mul-op98,Default/optimizer-FP32StateAdamWeightDecay/Mul-op98,kernel_graph0,{13904},13904,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13090,13875,0x12c223613000,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13083,456_Default/optimizer-FP32StateAdamWeightDecay/Mul-op91,Default/optimizer-FP32StateAdamWeightDecay/Mul-op91,kernel_graph0,{13872},13872,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13092,13097,0x12c223617200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13084,457_Default/optimizer-FP32StateAdamWeightDecay/Square-op18,Default/optimizer-FP32StateAdamWeightDecay/Square-op18,kernel_graph0,{13095},13095,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13095,13879,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13084,458_Default/optimizer-FP32StateAdamWeightDecay/Mul-op93,Default/optimizer-FP32StateAdamWeightDecay/Mul-op93,kernel_graph0,{13876},13876,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13098,13843,0x12c223617200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13085,459_Default/optimizer-FP32StateAdamWeightDecay/Mul-op86,Default/optimizer-FP32StateAdamWeightDecay/Mul-op86,kernel_graph0,{13840},13840,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13100,13105,0x12c22361b400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13086,460_Default/optimizer-FP32StateAdamWeightDecay/Square-op17,Default/optimizer-FP32StateAdamWeightDecay/Square-op17,kernel_graph0,{13103},13103,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13103,13847,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13087,461_Default/optimizer-FP32StateAdamWeightDecay/Mul-op88,Default/optimizer-FP32StateAdamWeightDecay/Mul-op88,kernel_graph0,{13844},13844,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13106,13811,0x12c22361b400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13088,462_Default/optimizer-FP32StateAdamWeightDecay/Mul-op81,Default/optimizer-FP32StateAdamWeightDecay/Mul-op81,kernel_graph0,{13808},13808,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13108,13113,0x12c22361f600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13089,463_Default/optimizer-FP32StateAdamWeightDecay/Square-op16,Default/optimizer-FP32StateAdamWeightDecay/Square-op16,kernel_graph0,{13111},13111,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13111,13815,0x12c22360ee00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13090,464_Default/optimizer-FP32StateAdamWeightDecay/Mul-op83,Default/optimizer-FP32StateAdamWeightDecay/Mul-op83,kernel_graph0,{13812},13812,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13114,13779,0x12c241610200,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13091,465_Default/optimizer-FP32StateAdamWeightDecay/Mul-op76,Default/optimizer-FP32StateAdamWeightDecay/Mul-op76,kernel_graph0,{13776},13776,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13116,13121,0x12c260a10400,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13092,466_Default/optimizer-FP32StateAdamWeightDecay/Square-op15,Default/optimizer-FP32StateAdamWeightDecay/Square-op15,kernel_graph0,{13119},13119,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13119,13783,0x12c180014400,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13093,467_Default/optimizer-FP32StateAdamWeightDecay/Mul-op78,Default/optimizer-FP32StateAdamWeightDecay/Mul-op78,kernel_graph0,{13780},13780,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13122,13747,0x12c1f9c12600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13093,468_Default/optimizer-FP32StateAdamWeightDecay/Mul-op71,Default/optimizer-FP32StateAdamWeightDecay/Mul-op71,kernel_graph0,{13744},13744,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13124,13129,0x12c260a10400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13095,469_Default/optimizer-FP32StateAdamWeightDecay/Square-op14,Default/optimizer-FP32StateAdamWeightDecay/Square-op14,kernel_graph0,{13127},13127,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13127,13751,0x12c218a0ec00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13096,470_Default/optimizer-FP32StateAdamWeightDecay/Mul-op73,Default/optimizer-FP32StateAdamWeightDecay/Mul-op73,kernel_graph0,{13748},13748,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13130,13715,0x12c260a10400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13096,471_Default/optimizer-FP32StateAdamWeightDecay/Mul-op66,Default/optimizer-FP32StateAdamWeightDecay/Mul-op66,kernel_graph0,{13712},13712,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13132,13137,0x12c26b610600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13098,472_Default/optimizer-FP32StateAdamWeightDecay/Square-op13,Default/optimizer-FP32StateAdamWeightDecay/Square-op13,kernel_graph0,{13135},13135,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13135,13719,0x12c22f60f600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13099,473_Default/optimizer-FP32StateAdamWeightDecay/Mul-op68,Default/optimizer-FP32StateAdamWeightDecay/Mul-op68,kernel_graph0,{13716},13716,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13138,13683,0x12c26b610600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13099,474_Default/optimizer-FP32StateAdamWeightDecay/Mul-op61,Default/optimizer-FP32StateAdamWeightDecay/Mul-op61,kernel_graph0,{13680},13680,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13140,13145,0x12c276210800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13100,475_Default/optimizer-FP32StateAdamWeightDecay/Square-op12,Default/optimizer-FP32StateAdamWeightDecay/Square-op12,kernel_graph0,{13143},13143,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13143,13687,0x12c22361f600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13101,476_Default/optimizer-FP32StateAdamWeightDecay/Mul-op63,Default/optimizer-FP32StateAdamWeightDecay/Mul-op63,kernel_graph0,{13684},13684,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13146,13651,0x12c204812800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13101,477_Default/optimizer-FP32StateAdamWeightDecay/Mul-op56,Default/optimizer-FP32StateAdamWeightDecay/Mul-op56,kernel_graph0,{13648},13648,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13148,13153,0x12c1e3936400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13103,478_Default/optimizer-FP32StateAdamWeightDecay/Square-op11,Default/optimizer-FP32StateAdamWeightDecay/Square-op11,kernel_graph0,{13151},13151,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13151,13655,0x12c1df936200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13104,479_Default/optimizer-FP32StateAdamWeightDecay/Mul-op58,Default/optimizer-FP32StateAdamWeightDecay/Mul-op58,kernel_graph0,{13652},13652,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13154,13619,0x12c1e3936400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13104,480_Default/optimizer-FP32StateAdamWeightDecay/Mul-op51,Default/optimizer-FP32StateAdamWeightDecay/Mul-op51,kernel_graph0,{13616},13616,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13156,13161,0x12c19f414600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13106,481_Default/optimizer-FP32StateAdamWeightDecay/Square-op10,Default/optimizer-FP32StateAdamWeightDecay/Square-op10,kernel_graph0,{13159},13159,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13159,13623,0x12c1edc0de00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13107,482_Default/optimizer-FP32StateAdamWeightDecay/Mul-op53,Default/optimizer-FP32StateAdamWeightDecay/Mul-op53,kernel_graph0,{13620},13620,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13162,13587,0x12c19f414600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13107,483_Default/optimizer-FP32StateAdamWeightDecay/Mul-op46,Default/optimizer-FP32StateAdamWeightDecay/Mul-op46,kernel_graph0,{13584},13584,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13164,13169,0x12c276210800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13108,484_Default/optimizer-FP32StateAdamWeightDecay/Square-op9,Default/optimizer-FP32StateAdamWeightDecay/Square-op9,kernel_graph0,{13167},13167,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13167,13591,0x12c1db936000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13109,485_Default/optimizer-FP32StateAdamWeightDecay/Mul-op48,Default/optimizer-FP32StateAdamWeightDecay/Mul-op48,kernel_graph0,{13588},13588,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13170,13555,0x12c276210800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13111,486_Default/optimizer-FP32StateAdamWeightDecay/Mul-op41,Default/optimizer-FP32StateAdamWeightDecay/Mul-op41,kernel_graph0,{13552},13552,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13172,13177,0x12c27a210a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13111,487_Default/optimizer-FP32StateAdamWeightDecay/Square-op8,Default/optimizer-FP32StateAdamWeightDecay/Square-op8,kernel_graph0,{13175},13175,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13175,13559,0x12c23a20f800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13112,488_Default/optimizer-FP32StateAdamWeightDecay/Mul-op43,Default/optimizer-FP32StateAdamWeightDecay/Mul-op43,kernel_graph0,{13556},13556,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13178,13523,0x12c27a210a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13114,489_Default/optimizer-FP32StateAdamWeightDecay/Mul-op36,Default/optimizer-FP32StateAdamWeightDecay/Mul-op36,kernel_graph0,{13520},13520,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13180,13185,0x12c284e10c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13115,490_Default/optimizer-FP32StateAdamWeightDecay/Square-op7,Default/optimizer-FP32StateAdamWeightDecay/Square-op7,kernel_graph0,{13183},13183,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13183,13527,0x12c20de0ea00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13115,491_Default/optimizer-FP32StateAdamWeightDecay/Mul-op38,Default/optimizer-FP32StateAdamWeightDecay/Mul-op38,kernel_graph0,{13524},13524,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13186,13491,0x12c284e10c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13116,492_Default/optimizer-FP32StateAdamWeightDecay/Mul-op31,Default/optimizer-FP32StateAdamWeightDecay/Mul-op31,kernel_graph0,{13488},13488,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13188,13193,0x12c28fa10e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13117,493_Default/optimizer-FP32StateAdamWeightDecay/Square-op6,Default/optimizer-FP32StateAdamWeightDecay/Square-op6,kernel_graph0,{13191},13191,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13191,13495,0x12c1d0d35e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13117,494_Default/optimizer-FP32StateAdamWeightDecay/Mul-op33,Default/optimizer-FP32StateAdamWeightDecay/Mul-op33,kernel_graph0,{13492},13492,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13194,13459,0x12c28fa10e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13119,495_Default/optimizer-FP32StateAdamWeightDecay/Mul-op26,Default/optimizer-FP32StateAdamWeightDecay/Mul-op26,kernel_graph0,{13456},13456,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13196,13201,0x12c29a611000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13120,496_Default/optimizer-FP32StateAdamWeightDecay/Square-op5,Default/optimizer-FP32StateAdamWeightDecay/Square-op5,kernel_graph0,{13199},13199,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13199,13463,0x12c1c6135c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13120,497_Default/optimizer-FP32StateAdamWeightDecay/Mul-op28,Default/optimizer-FP32StateAdamWeightDecay/Mul-op28,kernel_graph0,{13460},13460,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13202,13427,0x12c29a611000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13122,498_Default/optimizer-FP32StateAdamWeightDecay/Mul-op21,Default/optimizer-FP32StateAdamWeightDecay/Mul-op21,kernel_graph0,{13424},13424,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13204,13209,0x12c29e611200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13123,499_Default/optimizer-FP32StateAdamWeightDecay/Square-op4,Default/optimizer-FP32StateAdamWeightDecay/Square-op4,kernel_graph0,{13207},13207,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13207,13431,0x12c1f5c12400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13123,500_Default/optimizer-FP32StateAdamWeightDecay/Mul-op23,Default/optimizer-FP32StateAdamWeightDecay/Mul-op23,kernel_graph0,{13428},13428,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13210,13395,0x12c29e611200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13124,501_Default/optimizer-FP32StateAdamWeightDecay/Mul-op16,Default/optimizer-FP32StateAdamWeightDecay/Mul-op16,kernel_graph0,{13392},13392,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13212,13217,0x12c2a2611400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13125,502_Default/optimizer-FP32StateAdamWeightDecay/Square-op3,Default/optimizer-FP32StateAdamWeightDecay/Square-op3,kernel_graph0,{13215},13215,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13215,13399,0x12c1e7936600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13125,503_Default/optimizer-FP32StateAdamWeightDecay/Mul-op18,Default/optimizer-FP32StateAdamWeightDecay/Mul-op18,kernel_graph0,{13396},13396,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13218,13363,0x12c2a2611400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13127,504_Default/optimizer-FP32StateAdamWeightDecay/Mul-op11,Default/optimizer-FP32StateAdamWeightDecay/Mul-op11,kernel_graph0,{13360},13360,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13220,13225,0x12c2a6611600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13128,505_Default/optimizer-FP32StateAdamWeightDecay/Square-op2,Default/optimizer-FP32StateAdamWeightDecay/Square-op2,kernel_graph0,{13223},13223,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13223,13367,0x12c1f1c12200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13128,506_Default/optimizer-FP32StateAdamWeightDecay/Mul-op13,Default/optimizer-FP32StateAdamWeightDecay/Mul-op13,kernel_graph0,{13364},13364,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13226,13331,0x12c2a6611600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13130,507_Default/optimizer-FP32StateAdamWeightDecay/Mul-op6,Default/optimizer-FP32StateAdamWeightDecay/Mul-op6,kernel_graph0,{13328},13328,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13228,13233,0x12c2aa611800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13131,508_Default/optimizer-FP32StateAdamWeightDecay/Square-op1,Default/optimizer-FP32StateAdamWeightDecay/Square-op1,kernel_graph0,{13231},13231,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13231,13335,0x12c208812a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13131,509_Default/optimizer-FP32StateAdamWeightDecay/Mul-op8,Default/optimizer-FP32StateAdamWeightDecay/Mul-op8,kernel_graph0,{13332},13332,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13234,13259,0x12c180008c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13132,510_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Cast-op0,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Cast-op0,kernel_graph0,{13252-13255-13257},13257,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13236,13299,0x12c2aa611800,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13133,511_Default/optimizer-FP32StateAdamWeightDecay/Mul-op1,Default/optimizer-FP32StateAdamWeightDecay/Mul-op1,kernel_graph0,{13296},13296,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13238,13243,0x12c2c9a11a00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13133,512_Default/optimizer-FP32StateAdamWeightDecay/Square-op0,Default/optimizer-FP32StateAdamWeightDecay/Square-op0,kernel_graph0,{13241},13241,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13241,13303,0x12c1a3414800,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13135,513_Default/optimizer-FP32StateAdamWeightDecay/Mul-op3,Default/optimizer-FP32StateAdamWeightDecay/Mul-op3,kernel_graph0,{13300},13300,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13244,13943,0x12c22e21f800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13135,514_Default/optimizer-FP32StateAdamWeightDecay/Mul-op101,Default/optimizer-FP32StateAdamWeightDecay/Mul-op101,kernel_graph0,{13940},13940,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13246,13251,0x12c22e223a00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13136,515_Default/optimizer-FP32StateAdamWeightDecay/Square-op20,Default/optimizer-FP32StateAdamWeightDecay/Square-op20,kernel_graph0,{13249},13249,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13249,13939,0x12c18000c000,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13137,516_Default/optimizer-FP32StateAdamWeightDecay/Mul-op103,Default/optimizer-FP32StateAdamWeightDecay/Mul-op103,kernel_graph0,{13936},13936,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13252,13254,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13138,518_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/GreaterEqual-op0,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/GreaterEqual-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13255,13260,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13254,522_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Greater-op0,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Greater-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13257,13263,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13255,523_Default/TensorMove-op1,Default/TensorMove-op1,kernel_graph0,{13261},13261,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13261,13266,0x12c180008c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13260,527_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Sub-op1,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Sub-op1,kernel_graph0,{13264},13264,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13264,13269,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13261,528_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Div-op1,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Div-op1,kernel_graph0,{13267},13267,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13267,13272,0x12c180008c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13262,529_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Mul-op1,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Mul-op1,kernel_graph0,{13270},13270,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13270,13275,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13262,530_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Cos-op0,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Cos-op0,kernel_graph0,{13273},13273,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13273,13278,0x12c180008c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13264,531_Default/learning_rate-CosineWithWarmUpLR/Add-op6,Default/learning_rate-CosineWithWarmUpLR/Add-op6,kernel_graph0,{13276},13276,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13276,13281,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13265,532_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Mul-op2,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Mul-op2,kernel_graph0,{13279},13279,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13279,13284,0x12c180008c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13265,533_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Maximum-op0,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Maximum-op0,kernel_graph0,{13282},13282,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13282,13287,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13267,534_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Mul-op3,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Mul-op3,kernel_graph0,{13285},13285,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13285,13290,0x12c180008c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13268,535_Default/learning_rate-CosineWithWarmUpLR/Add-op7,Default/learning_rate-CosineWithWarmUpLR/Add-op7,kernel_graph0,{13288},13288,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13288,13954,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13271,538_Default/optimizer-FP32StateAdamWeightDecay/Reshape-op0,Default/optimizer-FP32StateAdamWeightDecay/Reshape-op0,kernel_graph0,{13312-13344-13376-13408-13440-13472-13504-13536-13568-13600-13632-13664-13696-13728-13760-13792-13824-13856-13888-13920-13952},13952,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13292,13298,0x12c2c9a11a00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13274,540_Default/optimizer-FP32StateAdamWeightDecay/Mul-op0,Default/optimizer-FP32StateAdamWeightDecay/Mul-op0,kernel_graph0,{13296},13296,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13294,13302,0x12c2e8e11c00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13274,541_Default/optimizer-FP32StateAdamWeightDecay/Mul-op2,Default/optimizer-FP32StateAdamWeightDecay/Mul-op2,kernel_graph0,{13300},13300,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13296,13321,0x12c308211e00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13276,542_Default/learning_rate-CosineWithWarmUpLR/Add-op8,Default/learning_rate-CosineWithWarmUpLR/Add-op8,kernel_graph0,{13309-13319},13319,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13300,13319,0x12c2aa611800,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13277,543_Default/learning_rate-CosineWithWarmUpLR/Add-op9,Default/learning_rate-CosineWithWarmUpLR/Add-op9,kernel_graph0,{13304-13317},13317,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13304,13308,0x12c1a3414800,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13277,544_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op0,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op0,kernel_graph0,{13306},13306,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13306,13311,0x12c2c9a11a00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13279,545_Default/learning_rate-CosineWithWarmUpLR/Add-op10,Default/learning_rate-CosineWithWarmUpLR/Add-op10,kernel_graph0,{13309},13309,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13309,13314,0x12c2e8e11c00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13279,546_Default/optimizer-FP32StateAdamWeightDecay/Div-op0,Default/optimizer-FP32StateAdamWeightDecay/Div-op0,kernel_graph0,{13312},13312,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13312,13317,0x12c2c9a11a00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13280,547_Default/optimizer-FP32StateAdamWeightDecay/Mul-op4,Default/optimizer-FP32StateAdamWeightDecay/Mul-op4,kernel_graph0,{13315},13315,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13315,13323,0x12c2e8e11c00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13281,548_Default/optimizer-FP32StateAdamWeightDecay/Sub-op0,Default/optimizer-FP32StateAdamWeightDecay/Sub-op0,kernel_graph0,{13321},13321,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13324,13330,0x12c1a3414800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13285,552_Default/optimizer-FP32StateAdamWeightDecay/Mul-op5,Default/optimizer-FP32StateAdamWeightDecay/Mul-op5,kernel_graph0,{13328},13328,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13326,13334,0x12c1a7414a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13286,553_Default/optimizer-FP32StateAdamWeightDecay/Mul-op7,Default/optimizer-FP32StateAdamWeightDecay/Mul-op7,kernel_graph0,{13332},13332,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13328,13353,0x12c1ab414c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13287,554_Default/learning_rate-CosineWithWarmUpLR/Add-op11,Default/learning_rate-CosineWithWarmUpLR/Add-op11,kernel_graph0,{13341-13351},13351,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13332,13351,0x12c1a3414800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13288,555_Default/learning_rate-CosineWithWarmUpLR/Add-op12,Default/learning_rate-CosineWithWarmUpLR/Add-op12,kernel_graph0,{13336-13349},13349,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13336,13340,0x12c1a7414a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13289,556_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op1,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op1,kernel_graph0,{13338},13338,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13338,13343,0x12c208812a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13290,557_Default/learning_rate-CosineWithWarmUpLR/Add-op13,Default/learning_rate-CosineWithWarmUpLR/Add-op13,kernel_graph0,{13341},13341,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13341,13346,0x12c1a7414a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13291,558_Default/optimizer-FP32StateAdamWeightDecay/Div-op1,Default/optimizer-FP32StateAdamWeightDecay/Div-op1,kernel_graph0,{13344},13344,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13344,13349,0x12c208812a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13291,559_Default/optimizer-FP32StateAdamWeightDecay/Mul-op9,Default/optimizer-FP32StateAdamWeightDecay/Mul-op9,kernel_graph0,{13347},13347,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13347,13355,0x12c1a7414a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13291,560_Default/optimizer-FP32StateAdamWeightDecay/Sub-op1,Default/optimizer-FP32StateAdamWeightDecay/Sub-op1,kernel_graph0,{13353},13353,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13356,13362,0x12c208812a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13293,564_Default/optimizer-FP32StateAdamWeightDecay/Mul-op10,Default/optimizer-FP32StateAdamWeightDecay/Mul-op10,kernel_graph0,{13360},13360,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13358,13366,0x12c1a3414800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13293,565_Default/optimizer-FP32StateAdamWeightDecay/Mul-op12,Default/optimizer-FP32StateAdamWeightDecay/Mul-op12,kernel_graph0,{13364},13364,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13360,13385,0x12c1a7414a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13294,566_Default/learning_rate-CosineWithWarmUpLR/Add-op14,Default/learning_rate-CosineWithWarmUpLR/Add-op14,kernel_graph0,{13373-13383},13383,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13364,13383,0x12c208812a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13295,567_Default/learning_rate-CosineWithWarmUpLR/Add-op15,Default/learning_rate-CosineWithWarmUpLR/Add-op15,kernel_graph0,{13368-13381},13381,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13368,13372,0x12c1a3414800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13295,568_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op2,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op2,kernel_graph0,{13370},13370,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13370,13375,0x12c1f1c12200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13296,569_Default/learning_rate-CosineWithWarmUpLR/Add-op16,Default/learning_rate-CosineWithWarmUpLR/Add-op16,kernel_graph0,{13373},13373,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13373,13378,0x12c1a3414800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13297,570_Default/optimizer-FP32StateAdamWeightDecay/Div-op2,Default/optimizer-FP32StateAdamWeightDecay/Div-op2,kernel_graph0,{13376},13376,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13376,13381,0x12c1f1c12200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13299,571_Default/optimizer-FP32StateAdamWeightDecay/Mul-op14,Default/optimizer-FP32StateAdamWeightDecay/Mul-op14,kernel_graph0,{13379},13379,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13379,13387,0x12c1a3414800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13300,572_Default/optimizer-FP32StateAdamWeightDecay/Sub-op2,Default/optimizer-FP32StateAdamWeightDecay/Sub-op2,kernel_graph0,{13385},13385,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13388,13394,0x12c1f1c12200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13304,576_Default/optimizer-FP32StateAdamWeightDecay/Mul-op15,Default/optimizer-FP32StateAdamWeightDecay/Mul-op15,kernel_graph0,{13392},13392,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13390,13398,0x12c208812a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13305,577_Default/optimizer-FP32StateAdamWeightDecay/Mul-op17,Default/optimizer-FP32StateAdamWeightDecay/Mul-op17,kernel_graph0,{13396},13396,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13392,13417,0x12c1a3414800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13305,578_Default/learning_rate-CosineWithWarmUpLR/Add-op17,Default/learning_rate-CosineWithWarmUpLR/Add-op17,kernel_graph0,{13405-13415},13415,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13396,13415,0x12c1f1c12200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13306,579_Default/learning_rate-CosineWithWarmUpLR/Add-op18,Default/learning_rate-CosineWithWarmUpLR/Add-op18,kernel_graph0,{13400-13413},13413,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13400,13404,0x12c208812a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13307,580_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op3,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op3,kernel_graph0,{13402},13402,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13402,13407,0x12c1e7936600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13308,581_Default/learning_rate-CosineWithWarmUpLR/Add-op19,Default/learning_rate-CosineWithWarmUpLR/Add-op19,kernel_graph0,{13405},13405,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13405,13410,0x12c208812a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13309,582_Default/optimizer-FP32StateAdamWeightDecay/Div-op3,Default/optimizer-FP32StateAdamWeightDecay/Div-op3,kernel_graph0,{13408},13408,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13408,13413,0x12c1e7936600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13310,583_Default/optimizer-FP32StateAdamWeightDecay/Mul-op19,Default/optimizer-FP32StateAdamWeightDecay/Mul-op19,kernel_graph0,{13411},13411,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13411,13419,0x12c208812a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13311,584_Default/optimizer-FP32StateAdamWeightDecay/Sub-op3,Default/optimizer-FP32StateAdamWeightDecay/Sub-op3,kernel_graph0,{13417},13417,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13420,13426,0x12c1f1c12200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13315,588_Default/optimizer-FP32StateAdamWeightDecay/Mul-op20,Default/optimizer-FP32StateAdamWeightDecay/Mul-op20,kernel_graph0,{13424},13424,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13422,13430,0x12c208812a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13316,589_Default/optimizer-FP32StateAdamWeightDecay/Mul-op22,Default/optimizer-FP32StateAdamWeightDecay/Mul-op22,kernel_graph0,{13428},13428,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13424,13449,0x12c1e7936600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13316,590_Default/learning_rate-CosineWithWarmUpLR/Add-op20,Default/learning_rate-CosineWithWarmUpLR/Add-op20,kernel_graph0,{13437-13447},13447,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13428,13447,0x12c1f1c12200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13317,591_Default/learning_rate-CosineWithWarmUpLR/Add-op21,Default/learning_rate-CosineWithWarmUpLR/Add-op21,kernel_graph0,{13432-13445},13445,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13432,13436,0x12c1f5c12400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13318,592_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op4,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op4,kernel_graph0,{13434},13434,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13434,13439,0x12c208812a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13318,593_Default/learning_rate-CosineWithWarmUpLR/Add-op22,Default/learning_rate-CosineWithWarmUpLR/Add-op22,kernel_graph0,{13437},13437,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13437,13442,0x12c1f5c12400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13318,594_Default/optimizer-FP32StateAdamWeightDecay/Div-op4,Default/optimizer-FP32StateAdamWeightDecay/Div-op4,kernel_graph0,{13440},13440,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13440,13445,0x12c208812a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13318,595_Default/optimizer-FP32StateAdamWeightDecay/Mul-op24,Default/optimizer-FP32StateAdamWeightDecay/Mul-op24,kernel_graph0,{13443},13443,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13443,13451,0x12c1f5c12400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13319,596_Default/optimizer-FP32StateAdamWeightDecay/Sub-op4,Default/optimizer-FP32StateAdamWeightDecay/Sub-op4,kernel_graph0,{13449},13449,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13452,13458,0x12c1a3414800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13320,600_Default/optimizer-FP32StateAdamWeightDecay/Mul-op25,Default/optimizer-FP32StateAdamWeightDecay/Mul-op25,kernel_graph0,{13456},13456,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13454,13462,0x12c1ae014a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13321,601_Default/optimizer-FP32StateAdamWeightDecay/Mul-op27,Default/optimizer-FP32StateAdamWeightDecay/Mul-op27,kernel_graph0,{13460},13460,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13456,13481,0x12c1b8c14c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13322,602_Default/learning_rate-CosineWithWarmUpLR/Add-op23,Default/learning_rate-CosineWithWarmUpLR/Add-op23,kernel_graph0,{13469-13479},13479,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13460,13479,0x12c1a3414800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13322,603_Default/learning_rate-CosineWithWarmUpLR/Add-op24,Default/learning_rate-CosineWithWarmUpLR/Add-op24,kernel_graph0,{13464-13477},13477,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13464,13468,0x12c1ae014a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13322,604_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op5,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op5,kernel_graph0,{13466},13466,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13466,13471,0x12c1c3814e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13323,605_Default/learning_rate-CosineWithWarmUpLR/Add-op25,Default/learning_rate-CosineWithWarmUpLR/Add-op25,kernel_graph0,{13469},13469,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13469,13474,0x12c1ae014a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13324,606_Default/optimizer-FP32StateAdamWeightDecay/Div-op5,Default/optimizer-FP32StateAdamWeightDecay/Div-op5,kernel_graph0,{13472},13472,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13472,13477,0x12c1c3814e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13325,607_Default/optimizer-FP32StateAdamWeightDecay/Mul-op29,Default/optimizer-FP32StateAdamWeightDecay/Mul-op29,kernel_graph0,{13475},13475,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13475,13483,0x12c1ae014a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13325,608_Default/optimizer-FP32StateAdamWeightDecay/Sub-op5,Default/optimizer-FP32StateAdamWeightDecay/Sub-op5,kernel_graph0,{13481},13481,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13484,13490,0x12c1a3414800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13328,612_Default/optimizer-FP32StateAdamWeightDecay/Mul-op30,Default/optimizer-FP32StateAdamWeightDecay/Mul-op30,kernel_graph0,{13488},13488,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13486,13494,0x12c1ae014a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13329,613_Default/optimizer-FP32StateAdamWeightDecay/Mul-op32,Default/optimizer-FP32StateAdamWeightDecay/Mul-op32,kernel_graph0,{13492},13492,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13488,13513,0x12c1b8c14c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13331,614_Default/learning_rate-CosineWithWarmUpLR/Add-op26,Default/learning_rate-CosineWithWarmUpLR/Add-op26,kernel_graph0,{13501-13511},13511,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13492,13511,0x12c1a3414800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13332,615_Default/learning_rate-CosineWithWarmUpLR/Add-op27,Default/learning_rate-CosineWithWarmUpLR/Add-op27,kernel_graph0,{13496-13509},13509,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13496,13500,0x12c1ae014a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13333,616_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op6,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op6,kernel_graph0,{13498},13498,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13498,13503,0x12c1c3814e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13335,617_Default/learning_rate-CosineWithWarmUpLR/Add-op28,Default/learning_rate-CosineWithWarmUpLR/Add-op28,kernel_graph0,{13501},13501,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13501,13506,0x12c1ae014a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13336,618_Default/optimizer-FP32StateAdamWeightDecay/Div-op6,Default/optimizer-FP32StateAdamWeightDecay/Div-op6,kernel_graph0,{13504},13504,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13504,13509,0x12c1c3814e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13337,619_Default/optimizer-FP32StateAdamWeightDecay/Mul-op34,Default/optimizer-FP32StateAdamWeightDecay/Mul-op34,kernel_graph0,{13507},13507,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13507,13515,0x12c1ae014a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13337,620_Default/optimizer-FP32StateAdamWeightDecay/Sub-op6,Default/optimizer-FP32StateAdamWeightDecay/Sub-op6,kernel_graph0,{13513},13513,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13516,13522,0x12c1a3414800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13341,624_Default/optimizer-FP32StateAdamWeightDecay/Mul-op35,Default/optimizer-FP32StateAdamWeightDecay/Mul-op35,kernel_graph0,{13520},13520,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13518,13526,0x12c1ae014a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13342,625_Default/optimizer-FP32StateAdamWeightDecay/Mul-op37,Default/optimizer-FP32StateAdamWeightDecay/Mul-op37,kernel_graph0,{13524},13524,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13520,13545,0x12c1b8c14c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13343,626_Default/learning_rate-CosineWithWarmUpLR/Add-op29,Default/learning_rate-CosineWithWarmUpLR/Add-op29,kernel_graph0,{13533-13543},13543,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13524,13543,0x12c1a3414800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13344,627_Default/learning_rate-CosineWithWarmUpLR/Add-op30,Default/learning_rate-CosineWithWarmUpLR/Add-op30,kernel_graph0,{13528-13541},13541,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13528,13532,0x12c1ae014a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13345,628_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op7,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op7,kernel_graph0,{13530},13530,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13530,13535,0x12c208812a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13346,629_Default/learning_rate-CosineWithWarmUpLR/Add-op31,Default/learning_rate-CosineWithWarmUpLR/Add-op31,kernel_graph0,{13533},13533,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13533,13538,0x12c1ae014a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13347,630_Default/optimizer-FP32StateAdamWeightDecay/Div-op7,Default/optimizer-FP32StateAdamWeightDecay/Div-op7,kernel_graph0,{13536},13536,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13536,13541,0x12c208812a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13348,631_Default/optimizer-FP32StateAdamWeightDecay/Mul-op39,Default/optimizer-FP32StateAdamWeightDecay/Mul-op39,kernel_graph0,{13539},13539,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13539,13547,0x12c1ae014a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13349,632_Default/optimizer-FP32StateAdamWeightDecay/Sub-op7,Default/optimizer-FP32StateAdamWeightDecay/Sub-op7,kernel_graph0,{13545},13545,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13548,13554,0x12c1e7936600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13350,636_Default/optimizer-FP32StateAdamWeightDecay/Mul-op40,Default/optimizer-FP32StateAdamWeightDecay/Mul-op40,kernel_graph0,{13552},13552,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13550,13558,0x12c1f1c12200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13351,637_Default/optimizer-FP32StateAdamWeightDecay/Mul-op42,Default/optimizer-FP32StateAdamWeightDecay/Mul-op42,kernel_graph0,{13556},13556,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13552,13577,0x12c1f5c12400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13352,638_Default/learning_rate-CosineWithWarmUpLR/Add-op32,Default/learning_rate-CosineWithWarmUpLR/Add-op32,kernel_graph0,{13565-13575},13575,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13556,13575,0x12c1e7936600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13352,639_Default/learning_rate-CosineWithWarmUpLR/Add-op33,Default/learning_rate-CosineWithWarmUpLR/Add-op33,kernel_graph0,{13560-13573},13573,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13560,13564,0x12c1f1c12200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13352,640_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op8,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op8,kernel_graph0,{13562},13562,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13562,13567,0x12c23a20f800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13353,641_Default/learning_rate-CosineWithWarmUpLR/Add-op34,Default/learning_rate-CosineWithWarmUpLR/Add-op34,kernel_graph0,{13565},13565,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13565,13570,0x12c1f1c12200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13354,642_Default/optimizer-FP32StateAdamWeightDecay/Div-op8,Default/optimizer-FP32StateAdamWeightDecay/Div-op8,kernel_graph0,{13568},13568,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13568,13573,0x12c23a20f800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13354,643_Default/optimizer-FP32StateAdamWeightDecay/Mul-op44,Default/optimizer-FP32StateAdamWeightDecay/Mul-op44,kernel_graph0,{13571},13571,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13571,13579,0x12c1f1c12200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13354,644_Default/optimizer-FP32StateAdamWeightDecay/Sub-op8,Default/optimizer-FP32StateAdamWeightDecay/Sub-op8,kernel_graph0,{13577},13577,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13580,13586,0x12c1e7936600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13358,648_Default/optimizer-FP32StateAdamWeightDecay/Mul-op45,Default/optimizer-FP32StateAdamWeightDecay/Mul-op45,kernel_graph0,{13584},13584,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13582,13590,0x12c23a20f800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13359,649_Default/optimizer-FP32StateAdamWeightDecay/Mul-op47,Default/optimizer-FP32StateAdamWeightDecay/Mul-op47,kernel_graph0,{13588},13588,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13584,13609,0x12c1f1c12200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13359,650_Default/learning_rate-CosineWithWarmUpLR/Add-op35,Default/learning_rate-CosineWithWarmUpLR/Add-op35,kernel_graph0,{13597-13607},13607,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13588,13607,0x12c1f5c12400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13360,651_Default/learning_rate-CosineWithWarmUpLR/Add-op36,Default/learning_rate-CosineWithWarmUpLR/Add-op36,kernel_graph0,{13592-13605},13605,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13592,13596,0x12c1e7936600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13361,652_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op9,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op9,kernel_graph0,{13594},13594,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13594,13599,0x12c23a20f800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13361,653_Default/learning_rate-CosineWithWarmUpLR/Add-op37,Default/learning_rate-CosineWithWarmUpLR/Add-op37,kernel_graph0,{13597},13597,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13597,13602,0x12c1e7936600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13364,654_Default/optimizer-FP32StateAdamWeightDecay/Div-op9,Default/optimizer-FP32StateAdamWeightDecay/Div-op9,kernel_graph0,{13600},13600,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13600,13605,0x12c23a20f800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13365,655_Default/optimizer-FP32StateAdamWeightDecay/Mul-op49,Default/optimizer-FP32StateAdamWeightDecay/Mul-op49,kernel_graph0,{13603},13603,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13603,13611,0x12c1e7936600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13365,656_Default/optimizer-FP32StateAdamWeightDecay/Sub-op9,Default/optimizer-FP32StateAdamWeightDecay/Sub-op9,kernel_graph0,{13609},13609,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13612,13618,0x12c1e7936600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13370,660_Default/optimizer-FP32StateAdamWeightDecay/Mul-op50,Default/optimizer-FP32StateAdamWeightDecay/Mul-op50,kernel_graph0,{13616},13616,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13614,13622,0x12c23a20f800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13371,661_Default/optimizer-FP32StateAdamWeightDecay/Mul-op52,Default/optimizer-FP32StateAdamWeightDecay/Mul-op52,kernel_graph0,{13620},13620,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13616,13641,0x12c1f1c12200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13372,662_Default/learning_rate-CosineWithWarmUpLR/Add-op38,Default/learning_rate-CosineWithWarmUpLR/Add-op38,kernel_graph0,{13629-13639},13639,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13620,13639,0x12c1f5c12400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13373,663_Default/learning_rate-CosineWithWarmUpLR/Add-op39,Default/learning_rate-CosineWithWarmUpLR/Add-op39,kernel_graph0,{13624-13637},13637,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13624,13628,0x12c23a20f800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13374,664_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op10,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op10,kernel_graph0,{13626},13626,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13626,13631,0x12c1e3936400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13375,665_Default/learning_rate-CosineWithWarmUpLR/Add-op40,Default/learning_rate-CosineWithWarmUpLR/Add-op40,kernel_graph0,{13629},13629,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13629,13634,0x12c23a20f800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13376,666_Default/optimizer-FP32StateAdamWeightDecay/Div-op10,Default/optimizer-FP32StateAdamWeightDecay/Div-op10,kernel_graph0,{13632},13632,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13632,13637,0x12c1e3936400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13377,667_Default/optimizer-FP32StateAdamWeightDecay/Mul-op54,Default/optimizer-FP32StateAdamWeightDecay/Mul-op54,kernel_graph0,{13635},13635,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13635,13643,0x12c23a20f800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13377,668_Default/optimizer-FP32StateAdamWeightDecay/Sub-op10,Default/optimizer-FP32StateAdamWeightDecay/Sub-op10,kernel_graph0,{13641},13641,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13644,13650,0x12c23a20f800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13381,672_Default/optimizer-FP32StateAdamWeightDecay/Mul-op55,Default/optimizer-FP32StateAdamWeightDecay/Mul-op55,kernel_graph0,{13648},13648,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13646,13654,0x12c1f1c12200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13382,673_Default/optimizer-FP32StateAdamWeightDecay/Mul-op57,Default/optimizer-FP32StateAdamWeightDecay/Mul-op57,kernel_graph0,{13652},13652,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13648,13673,0x12c1f5c12400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13382,674_Default/learning_rate-CosineWithWarmUpLR/Add-op41,Default/learning_rate-CosineWithWarmUpLR/Add-op41,kernel_graph0,{13661-13671},13671,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13652,13671,0x12c23a20f800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13382,675_Default/learning_rate-CosineWithWarmUpLR/Add-op42,Default/learning_rate-CosineWithWarmUpLR/Add-op42,kernel_graph0,{13656-13669},13669,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13656,13660,0x12c1f1c12200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13383,676_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op11,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op11,kernel_graph0,{13658},13658,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13658,13663,0x12c204812800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13384,677_Default/learning_rate-CosineWithWarmUpLR/Add-op43,Default/learning_rate-CosineWithWarmUpLR/Add-op43,kernel_graph0,{13661},13661,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13661,13666,0x12c1f1c12200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13384,678_Default/optimizer-FP32StateAdamWeightDecay/Div-op11,Default/optimizer-FP32StateAdamWeightDecay/Div-op11,kernel_graph0,{13664},13664,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13664,13669,0x12c204812800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13384,679_Default/optimizer-FP32StateAdamWeightDecay/Mul-op59,Default/optimizer-FP32StateAdamWeightDecay/Mul-op59,kernel_graph0,{13667},13667,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13667,13675,0x12c1f1c12200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13385,680_Default/optimizer-FP32StateAdamWeightDecay/Sub-op11,Default/optimizer-FP32StateAdamWeightDecay/Sub-op11,kernel_graph0,{13673},13673,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13676,13682,0x12c204812800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13388,684_Default/optimizer-FP32StateAdamWeightDecay/Mul-op60,Default/optimizer-FP32StateAdamWeightDecay/Mul-op60,kernel_graph0,{13680},13680,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13678,13686,0x12c19f414600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13389,685_Default/optimizer-FP32StateAdamWeightDecay/Mul-op62,Default/optimizer-FP32StateAdamWeightDecay/Mul-op62,kernel_graph0,{13684},13684,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13680,13705,0x12c1aa014800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13389,686_Default/learning_rate-CosineWithWarmUpLR/Add-op44,Default/learning_rate-CosineWithWarmUpLR/Add-op44,kernel_graph0,{13693-13703},13703,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13684,13703,0x12c204812800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13390,687_Default/learning_rate-CosineWithWarmUpLR/Add-op45,Default/learning_rate-CosineWithWarmUpLR/Add-op45,kernel_graph0,{13688-13701},13701,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13688,13692,0x12c19f414600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13391,688_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op12,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op12,kernel_graph0,{13690},13690,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13690,13695,0x12c22361f600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13391,689_Default/learning_rate-CosineWithWarmUpLR/Add-op46,Default/learning_rate-CosineWithWarmUpLR/Add-op46,kernel_graph0,{13693},13693,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13693,13698,0x12c19f414600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13392,690_Default/optimizer-FP32StateAdamWeightDecay/Div-op12,Default/optimizer-FP32StateAdamWeightDecay/Div-op12,kernel_graph0,{13696},13696,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13696,13701,0x12c22361f600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13393,691_Default/optimizer-FP32StateAdamWeightDecay/Mul-op64,Default/optimizer-FP32StateAdamWeightDecay/Mul-op64,kernel_graph0,{13699},13699,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13699,13707,0x12c19f414600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13393,692_Default/optimizer-FP32StateAdamWeightDecay/Sub-op12,Default/optimizer-FP32StateAdamWeightDecay/Sub-op12,kernel_graph0,{13705},13705,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13708,13714,0x12c22361f600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13400,696_Default/optimizer-FP32StateAdamWeightDecay/Mul-op65,Default/optimizer-FP32StateAdamWeightDecay/Mul-op65,kernel_graph0,{13712},13712,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13710,13718,0x12c204812800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13400,697_Default/optimizer-FP32StateAdamWeightDecay/Mul-op67,Default/optimizer-FP32StateAdamWeightDecay/Mul-op67,kernel_graph0,{13716},13716,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13712,13737,0x12c19f414600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13401,698_Default/learning_rate-CosineWithWarmUpLR/Add-op47,Default/learning_rate-CosineWithWarmUpLR/Add-op47,kernel_graph0,{13725-13735},13735,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13716,13735,0x12c22361f600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13401,699_Default/learning_rate-CosineWithWarmUpLR/Add-op48,Default/learning_rate-CosineWithWarmUpLR/Add-op48,kernel_graph0,{13720-13733},13733,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13720,13724,0x12c22e223a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13402,700_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op13,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op13,kernel_graph0,{13722},13722,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13722,13727,0x12c204812800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13403,701_Default/learning_rate-CosineWithWarmUpLR/Add-op49,Default/learning_rate-CosineWithWarmUpLR/Add-op49,kernel_graph0,{13725},13725,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13725,13730,0x12c22e223a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13404,702_Default/optimizer-FP32StateAdamWeightDecay/Div-op13,Default/optimizer-FP32StateAdamWeightDecay/Div-op13,kernel_graph0,{13728},13728,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13728,13733,0x12c204812800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13405,703_Default/optimizer-FP32StateAdamWeightDecay/Mul-op69,Default/optimizer-FP32StateAdamWeightDecay/Mul-op69,kernel_graph0,{13731},13731,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13731,13739,0x12c22e223a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13406,704_Default/optimizer-FP32StateAdamWeightDecay/Sub-op13,Default/optimizer-FP32StateAdamWeightDecay/Sub-op13,kernel_graph0,{13737},13737,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13740,13746,0x12c22361f600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13410,708_Default/optimizer-FP32StateAdamWeightDecay/Mul-op70,Default/optimizer-FP32StateAdamWeightDecay/Mul-op70,kernel_graph0,{13744},13744,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13742,13750,0x12c22e223a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13411,709_Default/optimizer-FP32StateAdamWeightDecay/Mul-op72,Default/optimizer-FP32StateAdamWeightDecay/Mul-op72,kernel_graph0,{13748},13748,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13744,13769,0x12c204812800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13412,710_Default/learning_rate-CosineWithWarmUpLR/Add-op50,Default/learning_rate-CosineWithWarmUpLR/Add-op50,kernel_graph0,{13757-13767},13767,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13748,13767,0x12c22361f600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13413,711_Default/learning_rate-CosineWithWarmUpLR/Add-op51,Default/learning_rate-CosineWithWarmUpLR/Add-op51,kernel_graph0,{13752-13765},13765,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13752,13756,0x12c1f1c12200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13413,712_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op14,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op14,kernel_graph0,{13754},13754,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13754,13759,0x12c22e223a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13414,713_Default/learning_rate-CosineWithWarmUpLR/Add-op52,Default/learning_rate-CosineWithWarmUpLR/Add-op52,kernel_graph0,{13757},13757,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13757,13762,0x12c1f1c12200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13414,714_Default/optimizer-FP32StateAdamWeightDecay/Div-op14,Default/optimizer-FP32StateAdamWeightDecay/Div-op14,kernel_graph0,{13760},13760,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13760,13765,0x12c22e223a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13414,715_Default/optimizer-FP32StateAdamWeightDecay/Mul-op74,Default/optimizer-FP32StateAdamWeightDecay/Mul-op74,kernel_graph0,{13763},13763,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13763,13771,0x12c1f1c12200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13415,716_Default/optimizer-FP32StateAdamWeightDecay/Sub-op14,Default/optimizer-FP32StateAdamWeightDecay/Sub-op14,kernel_graph0,{13769},13769,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13772,13778,0x12c1f1c12200,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13417,720_Default/optimizer-FP32StateAdamWeightDecay/Mul-op75,Default/optimizer-FP32StateAdamWeightDecay/Mul-op75,kernel_graph0,{13776},13776,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13774,13782,0x12c19f414600,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13418,721_Default/optimizer-FP32StateAdamWeightDecay/Mul-op77,Default/optimizer-FP32StateAdamWeightDecay/Mul-op77,kernel_graph0,{13780},13780,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13776,13801,0x12c1be814800,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13418,722_Default/learning_rate-CosineWithWarmUpLR/Add-op53,Default/learning_rate-CosineWithWarmUpLR/Add-op53,kernel_graph0,{13789-13799},13799,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13780,13799,0x12c1f1c12200,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13418,723_Default/learning_rate-CosineWithWarmUpLR/Add-op54,Default/learning_rate-CosineWithWarmUpLR/Add-op54,kernel_graph0,{13784-13797},13797,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13784,13788,0x12c180014400,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13420,724_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op15,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op15,kernel_graph0,{13786},13786,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13786,13791,0x12c19f414600,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13421,725_Default/learning_rate-CosineWithWarmUpLR/Add-op55,Default/learning_rate-CosineWithWarmUpLR/Add-op55,kernel_graph0,{13789},13789,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13789,13794,0x12c180014400,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13421,726_Default/optimizer-FP32StateAdamWeightDecay/Div-op15,Default/optimizer-FP32StateAdamWeightDecay/Div-op15,kernel_graph0,{13792},13792,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13792,13797,0x12c19f414600,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13422,727_Default/optimizer-FP32StateAdamWeightDecay/Mul-op79,Default/optimizer-FP32StateAdamWeightDecay/Mul-op79,kernel_graph0,{13795},13795,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13795,13803,0x12c180014400,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13423,728_Default/optimizer-FP32StateAdamWeightDecay/Sub-op15,Default/optimizer-FP32StateAdamWeightDecay/Sub-op15,kernel_graph0,{13801},13801,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13804,13810,0x12c22361f600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13425,732_Default/optimizer-FP32StateAdamWeightDecay/Mul-op80,Default/optimizer-FP32StateAdamWeightDecay/Mul-op80,kernel_graph0,{13808},13808,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13806,13814,0x12c223623800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13428,733_Default/optimizer-FP32StateAdamWeightDecay/Mul-op82,Default/optimizer-FP32StateAdamWeightDecay/Mul-op82,kernel_graph0,{13812},13812,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13808,13833,0x12c223627a00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13429,734_Default/learning_rate-CosineWithWarmUpLR/Add-op56,Default/learning_rate-CosineWithWarmUpLR/Add-op56,kernel_graph0,{13821-13831},13831,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13812,13831,0x12c22361b400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13429,735_Default/learning_rate-CosineWithWarmUpLR/Add-op57,Default/learning_rate-CosineWithWarmUpLR/Add-op57,kernel_graph0,{13816-13829},13829,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13816,13820,0x12c22361f600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13432,736_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op16,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op16,kernel_graph0,{13818},13818,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13818,13823,0x12c223623800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13433,737_Default/learning_rate-CosineWithWarmUpLR/Add-op58,Default/learning_rate-CosineWithWarmUpLR/Add-op58,kernel_graph0,{13821},13821,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13821,13826,0x12c22361f600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13433,738_Default/optimizer-FP32StateAdamWeightDecay/Div-op16,Default/optimizer-FP32StateAdamWeightDecay/Div-op16,kernel_graph0,{13824},13824,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13824,13829,0x12c223623800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13434,739_Default/optimizer-FP32StateAdamWeightDecay/Mul-op84,Default/optimizer-FP32StateAdamWeightDecay/Mul-op84,kernel_graph0,{13827},13827,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13827,13835,0x12c22361f600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13435,740_Default/optimizer-FP32StateAdamWeightDecay/Sub-op16,Default/optimizer-FP32StateAdamWeightDecay/Sub-op16,kernel_graph0,{13833},13833,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13836,13842,0x12c22361b400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13438,744_Default/optimizer-FP32StateAdamWeightDecay/Mul-op85,Default/optimizer-FP32StateAdamWeightDecay/Mul-op85,kernel_graph0,{13840},13840,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13838,13846,0x12c22361f600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13440,745_Default/optimizer-FP32StateAdamWeightDecay/Mul-op87,Default/optimizer-FP32StateAdamWeightDecay/Mul-op87,kernel_graph0,{13844},13844,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13840,13865,0x12c223623800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13441,746_Default/learning_rate-CosineWithWarmUpLR/Add-op59,Default/learning_rate-CosineWithWarmUpLR/Add-op59,kernel_graph0,{13853-13863},13863,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13844,13863,0x12c223617200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13441,747_Default/learning_rate-CosineWithWarmUpLR/Add-op60,Default/learning_rate-CosineWithWarmUpLR/Add-op60,kernel_graph0,{13848-13861},13861,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13848,13852,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13443,748_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op17,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op17,kernel_graph0,{13850},13850,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13850,13855,0x12c22361b400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13444,749_Default/learning_rate-CosineWithWarmUpLR/Add-op61,Default/learning_rate-CosineWithWarmUpLR/Add-op61,kernel_graph0,{13853},13853,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13853,13858,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13444,750_Default/optimizer-FP32StateAdamWeightDecay/Div-op17,Default/optimizer-FP32StateAdamWeightDecay/Div-op17,kernel_graph0,{13856},13856,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13856,13861,0x12c22361b400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13445,751_Default/optimizer-FP32StateAdamWeightDecay/Mul-op89,Default/optimizer-FP32StateAdamWeightDecay/Mul-op89,kernel_graph0,{13859},13859,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13859,13867,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13446,752_Default/optimizer-FP32StateAdamWeightDecay/Sub-op17,Default/optimizer-FP32StateAdamWeightDecay/Sub-op17,kernel_graph0,{13865},13865,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13868,13874,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13448,756_Default/optimizer-FP32StateAdamWeightDecay/Mul-op90,Default/optimizer-FP32StateAdamWeightDecay/Mul-op90,kernel_graph0,{13872},13872,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13870,13878,0x12c223617200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13448,757_Default/optimizer-FP32StateAdamWeightDecay/Mul-op92,Default/optimizer-FP32StateAdamWeightDecay/Mul-op92,kernel_graph0,{13876},13876,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13872,13897,0x12c22361b400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13448,758_Default/learning_rate-CosineWithWarmUpLR/Add-op62,Default/learning_rate-CosineWithWarmUpLR/Add-op62,kernel_graph0,{13885-13895},13895,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13876,13895,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13449,759_Default/learning_rate-CosineWithWarmUpLR/Add-op63,Default/learning_rate-CosineWithWarmUpLR/Add-op63,kernel_graph0,{13880-13893},13893,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13880,13884,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13450,760_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op18,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op18,kernel_graph0,{13882},13882,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13882,13887,0x12c22361f600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13450,761_Default/learning_rate-CosineWithWarmUpLR/Add-op64,Default/learning_rate-CosineWithWarmUpLR/Add-op64,kernel_graph0,{13885},13885,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13885,13890,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13450,762_Default/optimizer-FP32StateAdamWeightDecay/Div-op18,Default/optimizer-FP32StateAdamWeightDecay/Div-op18,kernel_graph0,{13888},13888,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13888,13893,0x12c22361f600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13451,763_Default/optimizer-FP32StateAdamWeightDecay/Mul-op94,Default/optimizer-FP32StateAdamWeightDecay/Mul-op94,kernel_graph0,{13891},13891,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13891,13899,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13452,764_Default/optimizer-FP32StateAdamWeightDecay/Sub-op18,Default/optimizer-FP32StateAdamWeightDecay/Sub-op18,kernel_graph0,{13897},13897,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13900,13906,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13455,768_Default/optimizer-FP32StateAdamWeightDecay/Mul-op97,Default/optimizer-FP32StateAdamWeightDecay/Mul-op97,kernel_graph0,{13904},13904,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13902,13910,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13456,769_Default/optimizer-FP32StateAdamWeightDecay/Mul-op95,Default/optimizer-FP32StateAdamWeightDecay/Mul-op95,kernel_graph0,{13908},13908,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13904,13927,0x12c1f1c12200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13456,770_Default/learning_rate-CosineWithWarmUpLR/Add-op66,Default/learning_rate-CosineWithWarmUpLR/Add-op66,kernel_graph0,{13912-13925},13925,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13908,13929,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13457,771_Default/learning_rate-CosineWithWarmUpLR/Add-op65,Default/learning_rate-CosineWithWarmUpLR/Add-op65,kernel_graph0,{13917-13927},13927,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13912,13916,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13457,772_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op19,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op19,kernel_graph0,{13914},13914,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13914,13919,0x12c1f1c16400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13460,773_Default/learning_rate-CosineWithWarmUpLR/Add-op67,Default/learning_rate-CosineWithWarmUpLR/Add-op67,kernel_graph0,{13917},13917,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13917,13922,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13461,774_Default/optimizer-FP32StateAdamWeightDecay/Div-op19,Default/optimizer-FP32StateAdamWeightDecay/Div-op19,kernel_graph0,{13920},13920,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13920,13925,0x12c1f1c16400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13461,775_Default/optimizer-FP32StateAdamWeightDecay/Mul-op99,Default/optimizer-FP32StateAdamWeightDecay/Mul-op99,kernel_graph0,{13923},13923,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13923,13931,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13464,776_Default/optimizer-FP32StateAdamWeightDecay/Sub-op19,Default/optimizer-FP32StateAdamWeightDecay/Sub-op19,kernel_graph0,{13929},13929,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13932,13938,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13467,780_Default/optimizer-FP32StateAdamWeightDecay/Mul-op102,Default/optimizer-FP32StateAdamWeightDecay/Mul-op102,kernel_graph0,{13936},13936,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13934,13942,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13467,781_Default/optimizer-FP32StateAdamWeightDecay/Mul-op100,Default/optimizer-FP32StateAdamWeightDecay/Mul-op100,kernel_graph0,{13940},13940,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13936,13964,0x12c180010200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13469,782_Default/learning_rate-CosineWithWarmUpLR/Add-op69,Default/learning_rate-CosineWithWarmUpLR/Add-op69,kernel_graph0,{13944-13962},13962,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13940,13962,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13470,783_Default/learning_rate-CosineWithWarmUpLR/Add-op68,Default/learning_rate-CosineWithWarmUpLR/Add-op68,kernel_graph0,{13949-13960},13960,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13944,13948,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13470,784_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op20,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op20,kernel_graph0,{13946},13946,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13946,13951,0x12c18000c000,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13472,785_Default/learning_rate-CosineWithWarmUpLR/Add-op70,Default/learning_rate-CosineWithWarmUpLR/Add-op70,kernel_graph0,{13949},13949,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13949,13955,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13473,786_Default/optimizer-FP32StateAdamWeightDecay/Div-op20,Default/optimizer-FP32StateAdamWeightDecay/Div-op20,kernel_graph0,{13952},13952,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13952,13958,0x12c18000c000,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13473,787_Default/optimizer-FP32StateAdamWeightDecay/Mul-op104,Default/optimizer-FP32StateAdamWeightDecay/Mul-op104,kernel_graph0,{13956},13956,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13956,13960,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13475,788_Default/optimizer-FP32StateAdamWeightDecay/Sub-op20,Default/optimizer-FP32StateAdamWeightDecay/Sub-op20,kernel_graph0,{13958},13958,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13965,16264,0x12c180008c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13478,792_Default/TensorMove-op2,Default/TensorMove-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13969,16262,0x12c75d35ee00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/output_actor.cc,689,Other,13967,kernel_graph_0_OutputActor,UpdateOutputDeviceAddress,,{},,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 13975,14379,0x12c75d35f000,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Kernel,13974,0_Default/network-LlamaForCausalLM/StridedSlice-op0,inputs0,0_Default/network-LlamaForCausalLM/StridedSlice-op0,{13976-14377},14377,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,1,0, 13976,14958,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13974,0_Default/network-LlamaForCausalLM/StridedSlice-op0,inputs0,0_Default/network-LlamaForCausalLM/StridedSlice-op0,{13978-13980-14955},14955,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13978,13984,0x12c18000c000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13976,1_Default/network-LlamaForCausalLM/model-LlamaModel/tok_embeddings-LlamaEmbedding/Gather-op0,Default/network-LlamaForCausalLM/model-LlamaModel/tok_embeddings-LlamaEmbedding/Gather-op0,kernel_graph0,{13982},13982,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13980,13990,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,8704,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13977,2_Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/NotEqual-op0,Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/NotEqual-op0,kernel_graph0,{13985-13988},13988,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13982,14936,0x12c18800c200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13977,3_Default/network-LlamaForCausalLM/model-LlamaModel/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/Cast-op0,kernel_graph0,{13994-14123-14934},14934,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13985,14413,0x12c18000c000,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13977,4_Default/network-LlamaForCausalLM/Cast-op1,Default/network-LlamaForCausalLM/Cast-op1,kernel_graph0,{13986-14005-14366-14410},14410,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13988,14019,0x12c180014200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13977,6_Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Cast-op0,kernel_graph0,{14013-14017},14017,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13991,14862,0x12c180018400,0,DefaultEnhancedAscendMemoryPool,524800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13978,7_Default/network-LlamaForCausalLM/model-LlamaModel/freqs_mgr-FreqsMgr/StridedSlice-op0,Default/network-LlamaForCausalLM/model-LlamaModel/freqs_mgr-FreqsMgr/StridedSlice-op0,kernel_graph0,{13992-14056-14079-14209-14224-14597-14618-14839-14860},14860,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13994,14000,0x12c18c00c400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13979,9_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op0,kernel_graph0,{13998},13998,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13996,14004,0x12c19400c600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13979,10_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{14002},14002,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13997,14943,0x12c180098600,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13979,10_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{14939},14939,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 13998,14001,0x12c1800a0800,0,DefaultEnhancedAscendMemoryPool,16777728,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13979,10_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14002,14914,0x12c1800a0800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13980,11_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op1,kernel_graph0,{14027-14032-14047-14070-14884-14898-14912},14912,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14005,14013,0x12c180008800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13980,12_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/ReduceSum-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/ReduceSum-op0,kernel_graph0,{14011},14011,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14007,14009,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,14007,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14008,14010,0x12c180009200,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,14008,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14011,14424,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13981,13_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Add-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Add-op0,kernel_graph0,{14354-14421},14421,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14015,14020,0x12c1840a0a00,0,DefaultEnhancedAscendMemoryPool,2097664,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13982,15_Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/ExpandDims-op0,Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/ExpandDims-op0,kernel_graph0,{14017},14017,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14017,14023,0x12c1842a0c00,0,DefaultEnhancedAscendMemoryPool,16777728,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13983,16_Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Mul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Mul-op0,kernel_graph0,{14021},14021,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14021,14027,0x12c1852a0e00,0,DefaultEnhancedAscendMemoryPool,16777728,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13984,17_Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Sub-op0,Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Sub-op0,kernel_graph0,{14023-14025},14025,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14025,14820,0x12c1840a0a00,0,DefaultEnhancedAscendMemoryPool,8389120,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13986,19_Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Cast-op1,kernel_graph0,{14108-14255-14573-14814},14814,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14029,14908,0x12c1848a0c00,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13987,21_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/Cast-op0,kernel_graph0,{14032-14905},14905,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14031,14039,0x12c18c00c400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13988,22_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,kernel_graph0,{14034-14037},14037,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14032,14034,0x12c1868a0e00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13988,22_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14036,14043,0x12c19000c600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13989,24_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,kernel_graph0,{14041},14041,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14037,14040,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13989,24_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14041,14060,0x12c19400c800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13991,25_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op2,kernel_graph0,{14056-14058},14058,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14044,14923,0x12c18c00c400,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13992,26_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/Cast-op0,kernel_graph0,{14047-14920},14920,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14046,14054,0x12c18e00c600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13992,27_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,kernel_graph0,{14049-14052},14052,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14047,14049,0x12c1868a0e00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13992,27_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14051,14063,0x12c19c00ca00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13992,29_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,kernel_graph0,{14061},14061,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14052,14055,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13992,29_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14056,14099,0x12c1a000cc00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13992,30_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op2,kernel_graph0,{14097},14097,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14058,14096,0x12c1a800ce00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13993,31_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op1,kernel_graph0,{14094},14094,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14061,14083,0x12c18e00c600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13994,32_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op0,kernel_graph0,{14079-14081},14081,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14064,14865,0x12c1868a0e00,0,DefaultEnhancedAscendMemoryPool,524800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13995,33_Default/network-LlamaForCausalLM/model-LlamaModel/freqs_mgr-FreqsMgr/StridedSlice-op1,Default/network-LlamaForCausalLM/model-LlamaModel/freqs_mgr-FreqsMgr/StridedSlice-op1,kernel_graph0,{14065-14084-14094-14214-14229-14599-14620-14841-14863},14863,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14067,14894,0x12c19600c800,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13998,35_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/Cast-op0,kernel_graph0,{14070-14891},14891,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14069,14077,0x12c19800ca00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13999,36_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,kernel_graph0,{14072-14075},14075,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14070,14072,0x12c186921000,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,13999,36_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14074,14818,0x12c1b000d000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14002,38_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,kernel_graph0,{14108-14814},14814,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14075,14078,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14002,38_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14079,14089,0x12c19800ca00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14003,39_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op0,kernel_graph0,{14087},14087,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14081,14086,0x12c1b400d200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14003,40_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op0,kernel_graph0,{14084},14084,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14084,14090,0x12c18e00c600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14005,41_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op1,kernel_graph0,{14087},14087,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14087,14093,0x12c1b400d200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14006,42_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op0,kernel_graph0,{14091},14091,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14091,14816,0x12c18e00c600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14006,43_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op1,kernel_graph0,{14108-14814},14814,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14094,14100,0x12c19800ca00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14006,44_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op3,kernel_graph0,{14097},14097,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14097,14103,0x12c1a800ce00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14006,45_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op1,kernel_graph0,{14101},14101,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14101,14817,0x12c19800ca00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14007,46_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op3,kernel_graph0,{14108-14814},14814,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14104,14821,0x12c186921000,0,DefaultEnhancedAscendMemoryPool,8389120,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14007,47_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{14814},14814,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14105,14822,0x12c187121200,0,DefaultEnhancedAscendMemoryPool,8389120,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14007,47_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{14814},14814,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14106,14823,0x12c180008800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14007,47_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{14814},14814,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14107,14824,0x12c19c00cc00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14007,47_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{14112-14814},14814,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14108,14110,0x12c1a000ce00,0,DefaultEnhancedAscendMemoryPool,84936192,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14007,47_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14111,14794,0x12c1a000ce00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14008,48_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,kernel_graph0,{14114-14119-14792},14792,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14112,14114,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14008,48_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14116,14802,0x12c19200c800,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14010,50_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/Cast-op0,kernel_graph0,{14119-14800},14800,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14118,14125,0x12c1a400d000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14011,51_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,kernel_graph0,{14121-14123},14123,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14119,14121,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14011,51_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14123,14774,0x12c1a800d200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14013,53_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/Add-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/Add-op0,kernel_graph0,{14126-14164-14772},14772,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14126,14132,0x12c1b400d200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14014,54_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op0,kernel_graph0,{14130},14130,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14128,14136,0x12c1bc00d400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14015,55_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{14134},14134,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14129,14781,0x12c187921400,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14015,55_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{14777},14777,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14130,14133,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,16777728,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14015,55_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14134,14753,0x12c1a400d000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14016,56_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op1,kernel_graph0,{14136-14141-14148-14737-14751},14751,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14138,14747,0x12c1b400d200,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14017,58_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/Cast-op0,kernel_graph0,{14141-14744},14744,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14140,14729,0x12c1b960d400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14018,59_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,kernel_graph0,{14143-14154-14727},14727,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14141,14143,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14018,59_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14145,14762,0x12c1c420d600,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14021,61_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/Cast-op0,kernel_graph0,{14148-14759},14759,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14147,14735,0x12c1c980d800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14022,62_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,kernel_graph0,{14151-14732},14732,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14148,14150,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14022,62_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14151,14725,0x12c1d440da00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14023,63_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/SiLU-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/SiLU-op0,kernel_graph0,{14152-14154-14723},14723,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14154,14712,0x12c1df00dc00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14024,65_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/Mul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/Mul-op0,kernel_graph0,{14155-14160-14710},14710,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14157,14720,0x12c1e9c0de00,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14026,67_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/Cast-op0,kernel_graph0,{14160-14718},14718,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14159,14166,0x12c1ef20e000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14027,68_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,kernel_graph0,{14162-14164},14164,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14160,14162,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14027,68_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14164,14692,0x12c1f320e200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14029,70_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/Add-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/Add-op1,kernel_graph0,{14167-14270-14690},14690,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14167,14173,0x12c1f720e400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14030,71_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op0,kernel_graph0,{14171},14171,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14169,14177,0x12c1ff20e600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14030,72_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{14175},14175,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14170,14699,0x12c187929600,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14030,72_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{14695},14695,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14171,14174,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,16777728,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14030,72_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14175,14670,0x12c1ef20e000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14032,73_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op1,kernel_graph0,{14177-14182-14197-14242-14640-14654-14668},14668,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14179,14664,0x12c1ac00d400,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14033,75_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/Cast-op0,kernel_graph0,{14182-14661},14661,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14181,14189,0x12c1f720e400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14034,76_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,kernel_graph0,{14184-14187},14187,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14182,14184,0x12c1ae00d600,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14034,76_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14186,14193,0x12c1fb20e600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14035,78_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,kernel_graph0,{14191},14191,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14187,14190,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14035,78_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14191,14213,0x12c1ff20e800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14037,79_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op2,kernel_graph0,{14209-14211},14211,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14194,14679,0x12c1f720e400,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14038,80_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/Cast-op0,kernel_graph0,{14197-14676},14676,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14196,14204,0x12c1f920e600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14038,81_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,kernel_graph0,{14199-14202},14202,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14197,14199,0x12c1ae00d600,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14038,81_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14201,14208,0x12c20720ea00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14042,83_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,kernel_graph0,{14206},14206,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14202,14205,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14042,83_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14206,14228,0x12c20b20ec00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14042,84_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op0,kernel_graph0,{14224-14226},14226,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14209,14219,0x12c21320ee00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14044,85_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op2,kernel_graph0,{14217},14217,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14211,14216,0x12c21b20f000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14045,86_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op1,kernel_graph0,{14214},14214,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14214,14220,0x12c1f920e600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14045,87_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op3,kernel_graph0,{14217},14217,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14217,14223,0x12c20120e800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14047,88_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op1,kernel_graph0,{14221},14221,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14221,14576,0x12c1f920e600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14048,89_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op3,kernel_graph0,{14255-14573},14573,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14224,14234,0x12c1fd20e800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14049,90_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op0,kernel_graph0,{14232},14232,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14226,14231,0x12c21320ee00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14050,91_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op0,kernel_graph0,{14229},14229,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14229,14235,0x12c20520ea00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14052,92_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op1,kernel_graph0,{14232},14232,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14232,14238,0x12c20d20ec00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14053,93_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op0,kernel_graph0,{14236},14236,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14236,14575,0x12c1fd20e800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14053,94_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op1,kernel_graph0,{14255-14573},14573,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14239,14650,0x12c20120ea00,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14056,95_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/Cast-op0,kernel_graph0,{14242-14647},14647,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14241,14249,0x12c20320ec00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14056,96_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,kernel_graph0,{14244-14247},14247,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14242,14244,0x12c1ae00d600,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14056,96_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14246,14577,0x12c20720ee00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14058,98_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,kernel_graph0,{14255-14573},14573,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14247,14250,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14058,98_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14251,14579,0x12c1ae00d600,0,DefaultEnhancedAscendMemoryPool,8389120,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14058,99_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{14573},14573,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14252,14580,0x12c1ae80d800,0,DefaultEnhancedAscendMemoryPool,8389120,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14058,99_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{14573},14573,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14253,14581,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14058,99_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{14573},14573,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14254,14582,0x12c20320ec00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14058,99_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{14259-14573},14573,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14255,14257,0x12c20b20f000,0,DefaultEnhancedAscendMemoryPool,84936192,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14058,99_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14258,14553,0x12c20b20f000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14059,100_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,kernel_graph0,{14261-14266-14551},14551,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14259,14261,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14059,100_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14263,14561,0x12c20f20f200,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14061,102_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/Cast-op0,kernel_graph0,{14266-14559},14559,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14265,14272,0x12c21120f400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14062,103_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,kernel_graph0,{14268-14270},14270,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14266,14268,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14062,103_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14270,14528,0x12c21520f600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14064,105_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/Add-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/Add-op0,kernel_graph0,{14273-14311-14526},14526,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14273,14279,0x12c21920f800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14065,106_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op0,kernel_graph0,{14277},14277,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14275,14283,0x12c22120fa00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14065,107_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{14281},14281,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14276,14535,0x12c187931800,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14065,107_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{14531},14531,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14277,14280,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,16777728,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14065,107_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14281,14507,0x12c21120f400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14065,108_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op1,kernel_graph0,{14283-14288-14295-14491-14505},14505,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14285,14501,0x12c21920f800,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14065,110_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/Cast-op0,kernel_graph0,{14288-14498},14498,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14287,14483,0x12c21e80fa00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14065,111_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,kernel_graph0,{14290-14301-14481},14481,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14288,14290,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14065,111_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14292,14516,0x12c22940fc00,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14067,113_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/Cast-op0,kernel_graph0,{14295-14513},14513,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14294,14489,0x12c22ea0fe00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14068,114_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,kernel_graph0,{14298-14486},14486,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14295,14297,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14068,114_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14298,14479,0x12c239610000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14068,115_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/SiLU-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/SiLU-op0,kernel_graph0,{14299-14301-14477},14477,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14301,14466,0x12c244210200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14071,117_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/Mul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/Mul-op0,kernel_graph0,{14302-14307-14464},14464,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14304,14474,0x12c24ee10400,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14072,119_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/Cast-op0,kernel_graph0,{14307-14472},14472,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14306,14313,0x12c254410600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14073,120_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,kernel_graph0,{14309-14311},14311,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14307,14309,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14073,120_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14311,14445,0x12c258410800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14076,122_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/Add-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/Add-op1,kernel_graph0,{14314-14443},14443,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14314,14320,0x12c25c410a00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14076,123_Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Cast-op0,kernel_graph0,{14318},14318,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14316,14324,0x12c264410c00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14079,124_Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{14322},14322,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14317,14452,0x12c187939a00,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14079,124_Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{14448},14448,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14318,14321,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,16777728,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14079,124_Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14322,14428,0x12c254410600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14080,125_Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Cast-op1,kernel_graph0,{14324-14329-14426},14426,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14326,14437,0x12c25c410a00,0,DefaultEnhancedAscendMemoryPool,262144512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14081,127_Default/network-LlamaForCausalLM/lm_head-Linear/Cast-op0,Default/network-LlamaForCausalLM/lm_head-Linear/Cast-op0,kernel_graph0,{14329-14434},14434,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14328,14334,0x12c26be10c00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14082,128_Default/network-LlamaForCausalLM/lm_head-Linear/MatMul-op0,Default/network-LlamaForCausalLM/lm_head-Linear/MatMul-op0,kernel_graph0,{14332},14332,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14329,14331,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14082,128_Default/network-LlamaForCausalLM/lm_head-Linear/MatMul-op0,Default/network-LlamaForCausalLM/lm_head-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14332,14341,0x12c28b210e00,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14082,129_Default/network-LlamaForCausalLM/Cast-op0,Default/network-LlamaForCausalLM/Cast-op0,kernel_graph0,{14336-14339},14339,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14335,14342,0x12c187941c00,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14084,130_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/ReduceMax-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/ReduceMax-op0,kernel_graph0,{14339},14339,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14336,14338,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14084,130_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/ReduceMax-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/ReduceMax-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14339,14375,0x12c2c9a11000,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14084,131_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Sub-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Sub-op0,kernel_graph0,{14345-14373},14373,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14343,14356,0x12c180009200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14085,132_Default/Mul-op0,Default/Mul-op0,kernel_graph0,{14354},14354,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14345,14353,0x12c26be10c00,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14086,133_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Exp-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Exp-op0,kernel_graph0,{14347},14347,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14347,14372,0x12c187941c00,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14087,134_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/ReduceSum-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/ReduceSum-op0,kernel_graph0,{14370},14370,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14349,14351,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,14349,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14350,14352,0x12c180009c00,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,14350,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14354,14360,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14088,135_Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_RealDiv/RealDiv-op0,Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_RealDiv/RealDiv-op0,kernel_graph0,{14358},14358,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14357,14365,0x12c180009200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14088,136_Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_RealDiv/SumExt-op0,Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_RealDiv/SumExt-op0,kernel_graph0,{14361-14363},14363,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14358,14361,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14088,136_Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_RealDiv/SumExt-op0,Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_RealDiv/SumExt-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14363,14368,0x12c187949e00,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14092,138_Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_ReduceSum/Tile-op0,Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_ReduceSum/Tile-op0,kernel_graph0,{14366},14366,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14366,14392,0x12c187952000,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14092,139_Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_Mul/Mul-op0,Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_Mul/Mul-op0,kernel_graph0,{14368-14389},14389,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14370,14376,0x12c187949e00,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14095,141_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Log-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Log-op0,kernel_graph0,{14373},14373,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14373,14398,0x12c26be10c00,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14095,142_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Sub-op1,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Sub-op1,kernel_graph0,{14384-14396},14396,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14377,14383,0x12c187941c00,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14097,143_Default/network-LlamaForCausalLM/StridedSlice-op1,Default/network-LlamaForCausalLM/StridedSlice-op1,kernel_graph0,{14379-14381},14381,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14381,14399,0x12c2aa610e00,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14098,145_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/OneHot-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/OneHot-op0,kernel_graph0,{14386-14396},14396,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14384,14388,0x12c2e8e11000,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14100,146_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Exp-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Exp-op0,kernel_graph0,{14386},14386,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14386,14391,0x12c327611200,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14101,147_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Sub-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Sub-op0,kernel_graph0,{14389},14389,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14389,14395,0x12c2e8e11000,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14102,148_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Mul-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Mul-op0,kernel_graph0,{14393},14393,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14393,14436,0x12c327611200,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14103,149_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op0,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op0,kernel_graph0,{14426-14434},14434,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14396,14402,0x12c2e8e11000,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14108,150_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Mul-op1,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Mul-op1,kernel_graph0,{14400},14400,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14400,14409,0x12c26be10c00,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14109,151_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Neg-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Neg-op0,kernel_graph0,{14403},14403,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14403,14412,0x12c187941c00,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14109,152_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/ReduceSum-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/ReduceSum-op0,kernel_graph0,{14410},14410,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14405,14407,0x12c180009200,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,14405,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14406,14408,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,14406,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14410,14420,0x12c187949e00,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14109,153_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Mul-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Mul-op0,kernel_graph0,{14414},14414,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14414,14423,0x12c180009200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14112,154_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/ReduceSum-op1,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/ReduceSum-op1,kernel_graph0,{14421},14421,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14416,14418,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,14416,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14417,14419,0x12c180009c00,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,14417,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14421,16257,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14112,155_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/RealDiv-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/RealDiv-op0,kernel_graph0,{16255},16255,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14425,14432,0x12c346a11400,0,DefaultEnhancedAscendMemoryPool,262144512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14113,156_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op29,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op29,kernel_graph0,{14430},14430,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14426,14429,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14113,156_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op29,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op29,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14430,15011,0x12c26be10c00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14113,157_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op29,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op29,kernel_graph0,{15009},15009,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14433,14442,0x12c254410600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14114,158_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op0,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op0,kernel_graph0,{14438-14440},14440,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14434,14438,0x12c346a11400,0,DefaultEnhancedAscendMemoryPool,155190272,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14114,158_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op0,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14440,14450,0x12c25c410a00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14116,160_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op1,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op1,kernel_graph0,{14448},14448,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14443,14451,0x12c28b210e00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14117,161_Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Cast-op2,kernel_graph0,{14448},14448,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14446,14456,0x12c254410600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14117,162_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op0,kernel_graph0,{14454},14454,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14447,15026,0x12c18000c000,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14117,162_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op0,kernel_graph0,{15024},15024,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14448,14453,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,16778752,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14117,162_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14454,14543,0x12c25c410800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14119,163_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op0,kernel_graph0,{14458-14464-14472-14540},14540,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14457,14462,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14120,164_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Mul-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Mul-op0,kernel_graph0,{14460},14460,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14460,15226,0x12c180009200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14121,166_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Maximum-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Maximum-op0,kernel_graph0,{15224},15224,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14463,14470,0x12c254410600,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14122,167_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op27,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op27,kernel_graph0,{14468},14468,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14464,14467,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14122,167_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op27,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op27,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14468,15005,0x12c244210200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14123,168_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op27,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op27,kernel_graph0,{15003},15003,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14471,14484,0x12c260410a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14124,169_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op1,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op1,kernel_graph0,{14475-14477-14481},14481,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14472,14475,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14124,169_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op1,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14477,14500,0x12c24ee10400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14126,171_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op0,kernel_graph0,{14479-14491-14498},14498,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14481,14488,0x12c239610000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14127,173_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op1,kernel_graph0,{14484-14486},14486,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14486,14515,0x12c21e80fa00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14130,175_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/Grad_SiLU/SiLUGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/Grad_SiLU/SiLUGrad-op0,kernel_graph0,{14505-14513},14513,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14490,14496,0x12c260410a00,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14131,176_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op28,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op28,kernel_graph0,{14494},14494,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14491,14493,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14131,176_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op28,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op28,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14494,15008,0x12c22ea0fe00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14131,177_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op28,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op28,kernel_graph0,{15006},15006,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14497,14521,0x12c239610000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14134,178_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op2,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op2,kernel_graph0,{14502-14519},14519,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14498,14502,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14134,178_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op2,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14504,14511,0x12c21920f800,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14135,180_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op26,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op26,kernel_graph0,{14509},14509,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14505,14508,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14135,180_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op26,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op26,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14509,15002,0x12c260410a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14136,181_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op26,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op26,kernel_graph0,{15000},15000,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14512,14522,0x12c21120f400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14137,182_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op3,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op3,kernel_graph0,{14517-14519},14519,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14513,14517,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14137,182_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op3,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op3,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14519,14525,0x12c23d610200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14139,184_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/AddN-op0,kernel_graph0,{14523},14523,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14523,14533,0x12c24ee10400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14139,185_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op2,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op2,kernel_graph0,{14531},14531,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14526,14534,0x12c239610000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14141,186_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op2,kernel_graph0,{14531},14531,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14529,14539,0x12c21120f400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14142,187_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op1,kernel_graph0,{14537},14537,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14530,15020,0x12c180010200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14142,187_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op1,kernel_graph0,{15018},15018,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14531,14536,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,16778752,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14142,187_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14537,14542,0x12c239610000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14142,188_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op1,kernel_graph0,{14540},14540,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14540,14707,0x12c23d610200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14143,189_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/AddN-op0,kernel_graph0,{14545-14551-14559-14704},14704,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14544,14549,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14144,190_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Sub-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Sub-op0,kernel_graph0,{14547},14547,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14547,15249,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14146,192_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/LessEqual-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/LessEqual-op0,kernel_graph0,{15229-15247},15247,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14550,14557,0x12c241610400,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14146,193_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op25,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op25,kernel_graph0,{14555},14555,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14551,14554,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14146,193_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op25,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op25,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14555,14999,0x12c20b20f000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14148,194_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op25,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op25,kernel_graph0,{14997},14997,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14558,14567,0x12c239610000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14149,195_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op4,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op4,kernel_graph0,{14562-14565},14565,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14559,14562,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14149,195_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op4,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op4,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14564,14578,0x12c24ee10400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14151,197_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op0,kernel_graph0,{14573},14573,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14565,14568,0x12c180009a00,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14151,197_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14569,14590,0x12c239610000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14151,198_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,kernel_graph0,{14588},14588,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14570,14587,0x12c252e10600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14151,198_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,kernel_graph0,{14585},14585,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14571,14594,0x12c256e10800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14151,198_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,kernel_graph0,{14592},14592,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14572,14583,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14151,198_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14573,14584,0x12c20f20f200,0,DefaultEnhancedAscendMemoryPool,515969024,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14151,198_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14585,14601,0x12c1f920e600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14152,199_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op5,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op5,kernel_graph0,{14597-14599},14599,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14588,14622,0x12c20320ec00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14152,200_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op3,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op3,kernel_graph0,{14618-14620},14620,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14591,14649,0x12c239610000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14152,201_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op3,kernel_graph0,{14595-14640-14647},14647,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14592,14595,0x12c180009a00,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14152,201_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op3,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14597,14607,0x12c24ee10400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14154,203_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op4,kernel_graph0,{14605},14605,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14599,14604,0x12c256e10600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14155,204_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op5,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op5,kernel_graph0,{14602},14602,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14602,14608,0x12c1f920e600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14155,205_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op1,kernel_graph0,{14605},14605,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14605,14611,0x12c256e10600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14156,206_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op1,kernel_graph0,{14609},14609,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14609,14615,0x12c24ee10400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14157,207_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op6,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op6,kernel_graph0,{14613},14613,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14612,14663,0x12c1f920e600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14158,208_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op2,kernel_graph0,{14616-14654-14661},14661,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14613,14616,0x12c180009a00,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14158,208_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14618,14628,0x12c24ee10400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14160,210_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op2,kernel_graph0,{14626},14626,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14620,14625,0x12c256e10600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14161,211_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op3,kernel_graph0,{14623},14623,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14623,14629,0x12c20320ec00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14161,212_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op0,kernel_graph0,{14626},14626,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14626,14632,0x12c256e10600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14162,213_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op0,kernel_graph0,{14630},14630,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14630,14636,0x12c1fd20e800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14163,214_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op4,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op4,kernel_graph0,{14634},14634,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14633,14678,0x12c20320ec00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14164,215_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op1,kernel_graph0,{14637-14668-14676},14676,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14634,14637,0x12c180009a00,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14164,215_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14639,14645,0x12c241610400,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14165,217_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op24,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op24,kernel_graph0,{14643},14643,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14640,14642,0x12c1ae00d600,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14165,217_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op24,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op24,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14643,14996,0x12c1fd20e800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14167,218_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op24,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op24,kernel_graph0,{14994},14994,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14646,14686,0x12c20720ee00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14167,219_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op7,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op7,kernel_graph0,{14651-14682},14682,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14647,14651,0x12c1ae00d600,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14167,219_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op7,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op7,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14653,14659,0x12c20120ea00,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14169,221_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op23,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op23,kernel_graph0,{14657},14657,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14654,14656,0x12c1ae00d600,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14169,221_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op23,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op23,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14657,14993,0x12c239610000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14171,222_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op23,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op23,kernel_graph0,{14991},14991,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14660,14685,0x12c24ee10400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14172,223_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op6,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op6,kernel_graph0,{14665-14682},14682,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14661,14665,0x12c1ae00d600,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14172,223_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op6,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op6,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14667,14674,0x12c20120ea00,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14175,225_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op22,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op22,kernel_graph0,{14672},14672,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14668,14671,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14175,225_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op22,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op22,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14672,14990,0x12c1ef20e000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14176,226_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op22,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op22,kernel_graph0,{14988},14988,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14675,14684,0x12c1f920e600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14176,227_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op5,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op5,kernel_graph0,{14680-14682},14682,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14676,14680,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14176,227_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op5,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op5,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14682,14689,0x12c20120ea00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14178,229_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/AddN-op0,kernel_graph0,{14687},14687,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14687,14697,0x12c24ee10400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14179,230_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op7,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op7,kernel_graph0,{14695},14695,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14690,14698,0x12c256e10600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14180,231_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op2,kernel_graph0,{14695},14695,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14693,14703,0x12c1f320e200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14181,232_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op2,kernel_graph0,{14701},14701,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14694,15023,0x12c187931800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14181,232_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op2,kernel_graph0,{15021},15021,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14695,14700,0x12c25ee10800,0,DefaultEnhancedAscendMemoryPool,16778752,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14181,232_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14701,14706,0x12c20120ea00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14182,233_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op2,kernel_graph0,{14704},14704,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14704,14789,0x12c20520ec00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14183,234_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/AddN-op0,kernel_graph0,{14707-14710-14718-14786},14786,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14709,14716,0x12c23d610200,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14184,236_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op20,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op20,kernel_graph0,{14714},14714,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14710,14713,0x12c242c10400,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14184,236_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op20,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op20,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14714,14984,0x12c1df00dc00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14185,237_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op20,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op20,kernel_graph0,{14982},14982,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14717,14730,0x12c24ee10400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14187,238_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op8,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op8,kernel_graph0,{14721-14723-14727},14727,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14718,14721,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14187,238_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op8,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op8,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14723,14746,0x12c20f20f200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14188,240_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op6,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op6,kernel_graph0,{14725-14737-14744},14744,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14727,14734,0x12c1d440da00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14191,242_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op7,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op7,kernel_graph0,{14730-14732},14732,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14732,14761,0x12c1b960d400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14193,244_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/Grad_SiLU/SiLUGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/Grad_SiLU/SiLUGrad-op1,kernel_graph0,{14751-14759},14759,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14736,14742,0x12c1e9c0de00,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14194,245_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op21,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op21,kernel_graph0,{14740},14740,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14737,14739,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14194,245_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op21,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op21,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14740,14987,0x12c24ee10400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14195,246_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op21,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op21,kernel_graph0,{14985},14985,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14743,14767,0x12c20120ea00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14196,247_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op9,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op9,kernel_graph0,{14748-14765},14765,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14744,14748,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14196,247_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op9,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op9,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14750,14757,0x12c1b400d200,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14198,249_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op19,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op19,kernel_graph0,{14755},14755,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14751,14754,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14198,249_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op19,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op19,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14755,14981,0x12c1c980d800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14199,250_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op19,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op19,kernel_graph0,{14979},14979,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14758,14768,0x12c1a400d000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14199,251_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op10,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op10,kernel_graph0,{14763-14765},14765,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14759,14763,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14199,251_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op10,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op10,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14765,14771,0x12c1e9c0de00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14202,253_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/AddN-op0,kernel_graph0,{14769},14769,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14769,14779,0x12c1f320e200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14203,254_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op8,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op8,kernel_graph0,{14777},14777,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14772,14780,0x12c1d440da00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14203,255_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op2,kernel_graph0,{14777},14777,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14775,14785,0x12c1a400d000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14206,256_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op3,kernel_graph0,{14783},14783,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14776,15014,0x12c187929600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14206,256_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op3,kernel_graph0,{15012},15012,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14777,14782,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,16778752,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14206,256_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op3,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14783,14788,0x12c20120ea00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14206,257_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op3,kernel_graph0,{14786},14786,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14786,14951,0x12c1e9c0de00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14207,258_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/AddN-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/AddN-op1,kernel_graph0,{14789-14792-14800-14948},14948,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14791,14798,0x12c259a10600,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14209,260_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op18,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op18,kernel_graph0,{14796},14796,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14792,14795,0x12c1edc0e000,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14209,260_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op18,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op18,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14796,14978,0x12c25ba10800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14210,261_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op18,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op18,kernel_graph0,{14976},14976,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14799,14808,0x12c23d610200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14211,262_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op11,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op11,kernel_graph0,{14803-14806},14806,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14800,14803,0x12c1edc0e000,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14211,262_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op11,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op11,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14805,14819,0x12c1f320e200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14212,264_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op4,kernel_graph0,{14814},14814,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14806,14809,0x12c180009a00,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14212,264_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op4,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14810,14832,0x12c1f720e400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14212,265_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,kernel_graph0,{14830},14830,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14811,14829,0x12c23d610200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14212,265_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,kernel_graph0,{14827},14827,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14812,14836,0x12c20120ea00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14212,265_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,kernel_graph0,{14834},14834,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14813,14825,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14212,265_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14814,14826,0x12c20f20f200,0,DefaultEnhancedAscendMemoryPool,515969024,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14212,265_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14827,14843,0x12c18e00c600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14214,266_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op11,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op11,kernel_graph0,{14839-14841},14841,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14830,14866,0x12c1d440da00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14215,267_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op9,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op9,kernel_graph0,{14860-14863},14863,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14833,14893,0x12c20520ec00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14215,268_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op7,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op7,kernel_graph0,{14837-14884-14891},14891,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14834,14837,0x12c180009a00,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14215,268_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op7,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op7,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14839,14849,0x12c1f320e200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14217,270_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op10,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op10,kernel_graph0,{14847},14847,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14841,14846,0x12c20f20f200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14218,271_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op11,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op11,kernel_graph0,{14844},14844,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14844,14850,0x12c18e00c600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14218,272_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op3,kernel_graph0,{14847},14847,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14847,14853,0x12c20f20f200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14221,273_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op1,kernel_graph0,{14851},14851,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14851,14857,0x12c20120ea00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14222,274_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op12,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op12,kernel_graph0,{14855},14855,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14854,14907,0x12c23d610200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14222,275_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op6,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op6,kernel_graph0,{14858-14898-14905},14905,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14855,14858,0x12c180009a00,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14222,275_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op6,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op6,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14860,14872,0x12c18e00c600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14225,277_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op8,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op8,kernel_graph0,{14870},14870,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14863,14869,0x12c1f320e200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14225,278_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op9,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op9,kernel_graph0,{14867},14867,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14867,14873,0x12c1d440da00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14226,279_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op2,kernel_graph0,{14870},14870,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14870,14876,0x12c1f320e200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14227,280_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op0,kernel_graph0,{14874},14874,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14874,14880,0x12c20120ea00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14227,281_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op10,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op10,kernel_graph0,{14878},14878,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14877,14922,0x12c18e00c600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14228,282_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op5,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op5,kernel_graph0,{14881-14912-14920},14920,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14878,14881,0x12c180009a00,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14228,282_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op5,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op5,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14883,14889,0x12c20920ee00,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14230,284_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op17,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op17,kernel_graph0,{14887},14887,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14884,14886,0x12c1edc0e000,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14230,284_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op17,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op17,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14887,14975,0x12c20120ea00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14230,285_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op17,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op17,kernel_graph0,{14973},14973,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14890,14930,0x12c1f320e200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14232,286_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op14,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op14,kernel_graph0,{14895-14926},14926,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14891,14895,0x12c1edc0e000,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14232,286_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op14,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op14,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14897,14903,0x12c259a10600,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14233,288_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op16,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op16,kernel_graph0,{14901},14901,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14898,14900,0x12c1edc0e000,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14233,288_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op16,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op16,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14901,14972,0x12c1f720e400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14236,289_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op16,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op16,kernel_graph0,{14970},14970,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14904,14929,0x12c20520ec00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14237,290_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op13,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op13,kernel_graph0,{14909-14926},14926,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14905,14909,0x12c1edc0e000,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14237,290_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op13,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op13,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14911,14918,0x12c1fb20e600,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14239,292_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op15,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op15,kernel_graph0,{14916},14916,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14912,14915,0x12c1edc0e000,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14239,292_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op15,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op15,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14916,14969,0x12c23d610200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14240,293_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op15,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op15,kernel_graph0,{14967},14967,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14919,14928,0x12c1800a0800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14240,294_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op12,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op12,kernel_graph0,{14924-14926},14926,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14920,14924,0x12c1edc0e000,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14240,294_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op12,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op12,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14926,14933,0x12c1d440da00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14242,296_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/AddN-op0,kernel_graph0,{14931},14931,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14931,14941,0x12c20f20f200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14243,297_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op13,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op13,kernel_graph0,{14939},14939,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14934,14942,0x12c1d440da00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14243,298_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op2,kernel_graph0,{14939},14939,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14937,14947,0x12c21720f400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14244,299_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op4,kernel_graph0,{14945},14945,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14938,15017,0x12c180014400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14244,299_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op4,kernel_graph0,{15015},15015,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14939,14944,0x12c1edc0e000,0,DefaultEnhancedAscendMemoryPool,16778752,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14244,299_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op4,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14945,14950,0x12c1f320e200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14245,300_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op4,kernel_graph0,{14948},14948,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14948,14954,0x12c20520ec00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14247,301_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/AddN-op0,kernel_graph0,{14952},14952,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14952,14957,0x12c1d440da00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14248,302_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op14,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op14,kernel_graph0,{14955},14955,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14955,14966,0x12c20f20f200,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14248,303_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/tok_embeddings-LlamaEmbedding/Grad_Gather/UnsortedSegmentSum-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/tok_embeddings-LlamaEmbedding/Grad_Gather/UnsortedSegmentSum-op0,kernel_graph0,{14964},14964,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14959,14963,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14255,304_Default/Mul-op1,Default/Mul-op1,kernel_graph0,{14961},14961,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14961,15027,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14256,305_Default/Reciprocal-op0,Default/Reciprocal-op0,kernel_graph0,{14964-14967-14970-14973-14976-14979-14982-14985-14988-14991-14994-14997-15000-15003-15006-15009-15012-15015-15018-15021-15024},15024,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14964,15262,0x12c187935a00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14256,306_Default/Mul-op2,Default/Mul-op2,kernel_graph0,{15028-15188-15260},15260,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14967,15265,0x12c1f320e200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14256,307_Default/Mul-op3,Default/Mul-op3,kernel_graph0,{15036-15188-15263},15263,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14970,15268,0x12c1e9c0de00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14259,308_Default/Mul-op4,Default/Mul-op4,kernel_graph0,{15044-15188-15266},15266,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14973,15271,0x12c1f720e400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14260,309_Default/Mul-op5,Default/Mul-op5,kernel_graph0,{15052-15188-15269},15269,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14976,15274,0x12c23d610200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14260,310_Default/Mul-op6,Default/Mul-op6,kernel_graph0,{15060-15188-15272},15272,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14979,15277,0x12c1d440da00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14261,311_Default/Mul-op7,Default/Mul-op7,kernel_graph0,{15068-15188-15275},15275,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14982,15280,0x12c20f20f200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14261,312_Default/Mul-op8,Default/Mul-op8,kernel_graph0,{15076-15188-15278},15278,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14985,15283,0x12c1df00dc00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14262,313_Default/Mul-op9,Default/Mul-op9,kernel_graph0,{15084-15188-15281},15281,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14988,15286,0x12c180018600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14263,314_Default/Mul-op10,Default/Mul-op10,kernel_graph0,{15092-15188-15284},15284,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14991,15289,0x12c1edc0e000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14264,315_Default/Mul-op11,Default/Mul-op11,kernel_graph0,{15100-15188-15287},15287,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14994,15292,0x12c239610000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14264,316_Default/Mul-op12,Default/Mul-op12,kernel_graph0,{15108-15188-15290},15290,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 14997,15295,0x12c1fb20e600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14266,317_Default/Mul-op13,Default/Mul-op13,kernel_graph0,{15116-15188-15293},15293,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15000,15298,0x12c1ff20e800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14267,318_Default/Mul-op14,Default/Mul-op14,kernel_graph0,{15124-15188-15296},15296,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15003,15301,0x12c219e0f400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14267,319_Default/Mul-op15,Default/Mul-op15,kernel_graph0,{15132-15188-15299},15299,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15006,15304,0x12c241610400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14268,320_Default/Mul-op16,Default/Mul-op16,kernel_graph0,{15140-15188-15302},15302,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15009,15307,0x12c24c210600,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14269,321_Default/Mul-op17,Default/Mul-op17,kernel_graph0,{15148-15188-15305},15305,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15012,15310,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14270,322_Default/Mul-op18,Default/Mul-op18,kernel_graph0,{15156-15188-15308},15308,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15015,15313,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14271,323_Default/Mul-op19,Default/Mul-op19,kernel_graph0,{15164-15188-15311},15311,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15018,15316,0x12c180014400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14271,324_Default/Mul-op20,Default/Mul-op20,kernel_graph0,{15172-15188-15314},15314,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15021,15319,0x12c180010200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14273,325_Default/Mul-op21,Default/Mul-op21,kernel_graph0,{15180-15188-15317},15317,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15024,15322,0x12c1f1c0e200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14274,326_Default/Mul-op22,Default/Mul-op22,kernel_graph0,{15188-15190-15320},15320,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15028,15033,0x12c1a6d35c00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14274,327_Default/clip_grad_norm-ClipGradNorm/Square-op0,Default/clip_grad_norm-ClipGradNorm/Square-op0,kernel_graph0,{15031},15031,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15030,15200,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14277,328_Default/clip_grad_norm-ClipGradNorm/SumExt-op0,Default/clip_grad_norm-ClipGradNorm/SumExt-op0,kernel_graph0,{15034-15198},15198,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15031,15034,0x12c18000c000,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14277,328_Default/clip_grad_norm-ClipGradNorm/SumExt-op0,Default/clip_grad_norm-ClipGradNorm/SumExt-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15036,15041,0x12c209e0ea00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14278,330_Default/clip_grad_norm-ClipGradNorm/Square-op1,Default/clip_grad_norm-ClipGradNorm/Square-op1,kernel_graph0,{15039},15039,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15038,15201,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14278,331_Default/clip_grad_norm-ClipGradNorm/SumExt-op1,Default/clip_grad_norm-ClipGradNorm/SumExt-op1,kernel_graph0,{15042-15198},15198,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15039,15042,0x12c18000c000,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14278,331_Default/clip_grad_norm-ClipGradNorm/SumExt-op1,Default/clip_grad_norm-ClipGradNorm/SumExt-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15044,15049,0x12c209e0ea00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14282,333_Default/clip_grad_norm-ClipGradNorm/Square-op2,Default/clip_grad_norm-ClipGradNorm/Square-op2,kernel_graph0,{15047},15047,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15046,15202,0x12c180009a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14282,334_Default/clip_grad_norm-ClipGradNorm/SumExt-op2,Default/clip_grad_norm-ClipGradNorm/SumExt-op2,kernel_graph0,{15050-15198},15198,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15047,15050,0x12c18000c000,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14282,334_Default/clip_grad_norm-ClipGradNorm/SumExt-op2,Default/clip_grad_norm-ClipGradNorm/SumExt-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15052,15057,0x12c209e0ea00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14284,336_Default/clip_grad_norm-ClipGradNorm/Square-op3,Default/clip_grad_norm-ClipGradNorm/Square-op3,kernel_graph0,{15055},15055,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15054,15203,0x12c180009c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14285,337_Default/clip_grad_norm-ClipGradNorm/SumExt-op3,Default/clip_grad_norm-ClipGradNorm/SumExt-op3,kernel_graph0,{15058-15198},15198,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15055,15058,0x12c18000c000,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14285,337_Default/clip_grad_norm-ClipGradNorm/SumExt-op3,Default/clip_grad_norm-ClipGradNorm/SumExt-op3,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15060,15065,0x12c209e0ea00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14286,339_Default/clip_grad_norm-ClipGradNorm/Square-op4,Default/clip_grad_norm-ClipGradNorm/Square-op4,kernel_graph0,{15063},15063,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15062,15204,0x12c180009e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14288,340_Default/clip_grad_norm-ClipGradNorm/SumExt-op4,Default/clip_grad_norm-ClipGradNorm/SumExt-op4,kernel_graph0,{15066-15198},15198,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15063,15066,0x12c18000c000,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14288,340_Default/clip_grad_norm-ClipGradNorm/SumExt-op4,Default/clip_grad_norm-ClipGradNorm/SumExt-op4,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15068,15073,0x12c224a0f600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14289,342_Default/clip_grad_norm-ClipGradNorm/Square-op5,Default/clip_grad_norm-ClipGradNorm/Square-op5,kernel_graph0,{15071},15071,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15070,15205,0x12c18000a000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14290,343_Default/clip_grad_norm-ClipGradNorm/SumExt-op5,Default/clip_grad_norm-ClipGradNorm/SumExt-op5,kernel_graph0,{15074-15198},15198,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15071,15074,0x12c18000c000,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14290,343_Default/clip_grad_norm-ClipGradNorm/SumExt-op5,Default/clip_grad_norm-ClipGradNorm/SumExt-op5,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15076,15081,0x12c224a0f600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14292,345_Default/clip_grad_norm-ClipGradNorm/Square-op6,Default/clip_grad_norm-ClipGradNorm/Square-op6,kernel_graph0,{15079},15079,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15078,15206,0x12c18000a200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14293,346_Default/clip_grad_norm-ClipGradNorm/SumExt-op6,Default/clip_grad_norm-ClipGradNorm/SumExt-op6,kernel_graph0,{15082-15198},15198,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15079,15082,0x12c18000c000,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14293,346_Default/clip_grad_norm-ClipGradNorm/SumExt-op6,Default/clip_grad_norm-ClipGradNorm/SumExt-op6,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15084,15089,0x12c224a0f600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14295,348_Default/clip_grad_norm-ClipGradNorm/Square-op7,Default/clip_grad_norm-ClipGradNorm/Square-op7,kernel_graph0,{15087},15087,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15086,15207,0x12c18000a400,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14296,349_Default/clip_grad_norm-ClipGradNorm/SumExt-op7,Default/clip_grad_norm-ClipGradNorm/SumExt-op7,kernel_graph0,{15090-15198},15198,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15087,15090,0x12c18000c000,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14296,349_Default/clip_grad_norm-ClipGradNorm/SumExt-op7,Default/clip_grad_norm-ClipGradNorm/SumExt-op7,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15092,15097,0x12c209e0ea00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14298,351_Default/clip_grad_norm-ClipGradNorm/Square-op8,Default/clip_grad_norm-ClipGradNorm/Square-op8,kernel_graph0,{15095},15095,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15094,15208,0x12c18000a600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14299,352_Default/clip_grad_norm-ClipGradNorm/SumExt-op8,Default/clip_grad_norm-ClipGradNorm/SumExt-op8,kernel_graph0,{15098-15198},15198,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15095,15098,0x12c18000c000,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14299,352_Default/clip_grad_norm-ClipGradNorm/SumExt-op8,Default/clip_grad_norm-ClipGradNorm/SumExt-op8,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15100,15105,0x12c209e0ea00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14299,354_Default/clip_grad_norm-ClipGradNorm/Square-op9,Default/clip_grad_norm-ClipGradNorm/Square-op9,kernel_graph0,{15103},15103,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15102,15209,0x12c18000a800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14300,355_Default/clip_grad_norm-ClipGradNorm/SumExt-op9,Default/clip_grad_norm-ClipGradNorm/SumExt-op9,kernel_graph0,{15106-15198},15198,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15103,15106,0x12c18000c000,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14300,355_Default/clip_grad_norm-ClipGradNorm/SumExt-op9,Default/clip_grad_norm-ClipGradNorm/SumExt-op9,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15108,15113,0x12c209e0ea00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14302,357_Default/clip_grad_norm-ClipGradNorm/Square-op10,Default/clip_grad_norm-ClipGradNorm/Square-op10,kernel_graph0,{15111},15111,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15110,15210,0x12c18000aa00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14302,358_Default/clip_grad_norm-ClipGradNorm/SumExt-op10,Default/clip_grad_norm-ClipGradNorm/SumExt-op10,kernel_graph0,{15114-15198},15198,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15111,15114,0x12c18000c000,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14302,358_Default/clip_grad_norm-ClipGradNorm/SumExt-op10,Default/clip_grad_norm-ClipGradNorm/SumExt-op10,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15116,15121,0x12c209e0ea00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14304,360_Default/clip_grad_norm-ClipGradNorm/Square-op11,Default/clip_grad_norm-ClipGradNorm/Square-op11,kernel_graph0,{15119},15119,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15118,15211,0x12c18000ac00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14305,361_Default/clip_grad_norm-ClipGradNorm/SumExt-op11,Default/clip_grad_norm-ClipGradNorm/SumExt-op11,kernel_graph0,{15122-15198},15198,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15119,15122,0x12c18000c000,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14305,361_Default/clip_grad_norm-ClipGradNorm/SumExt-op11,Default/clip_grad_norm-ClipGradNorm/SumExt-op11,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15124,15129,0x12c224a0f600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14307,363_Default/clip_grad_norm-ClipGradNorm/Square-op12,Default/clip_grad_norm-ClipGradNorm/Square-op12,kernel_graph0,{15127},15127,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15126,15212,0x12c18000ae00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14308,364_Default/clip_grad_norm-ClipGradNorm/SumExt-op12,Default/clip_grad_norm-ClipGradNorm/SumExt-op12,kernel_graph0,{15130-15198},15198,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15127,15130,0x12c18000c000,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14308,364_Default/clip_grad_norm-ClipGradNorm/SumExt-op12,Default/clip_grad_norm-ClipGradNorm/SumExt-op12,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15132,15137,0x12c224a0f600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14310,366_Default/clip_grad_norm-ClipGradNorm/Square-op13,Default/clip_grad_norm-ClipGradNorm/Square-op13,kernel_graph0,{15135},15135,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15134,15213,0x12c18000b000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14311,367_Default/clip_grad_norm-ClipGradNorm/SumExt-op13,Default/clip_grad_norm-ClipGradNorm/SumExt-op13,kernel_graph0,{15138-15198},15198,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15135,15138,0x12c18000c000,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14311,367_Default/clip_grad_norm-ClipGradNorm/SumExt-op13,Default/clip_grad_norm-ClipGradNorm/SumExt-op13,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15140,15145,0x12c224a0f600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14312,369_Default/clip_grad_norm-ClipGradNorm/Square-op14,Default/clip_grad_norm-ClipGradNorm/Square-op14,kernel_graph0,{15143},15143,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15142,15214,0x12c18000b200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14314,370_Default/clip_grad_norm-ClipGradNorm/SumExt-op14,Default/clip_grad_norm-ClipGradNorm/SumExt-op14,kernel_graph0,{15146-15198},15198,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15143,15146,0x12c18000c000,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14314,370_Default/clip_grad_norm-ClipGradNorm/SumExt-op14,Default/clip_grad_norm-ClipGradNorm/SumExt-op14,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15148,15153,0x12c1a6d35c00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14315,372_Default/clip_grad_norm-ClipGradNorm/Square-op15,Default/clip_grad_norm-ClipGradNorm/Square-op15,kernel_graph0,{15151},15151,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15150,15215,0x12c18000b400,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14318,373_Default/clip_grad_norm-ClipGradNorm/SumExt-op15,Default/clip_grad_norm-ClipGradNorm/SumExt-op15,kernel_graph0,{15154-15198},15198,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15151,15154,0x12c18000c000,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14318,373_Default/clip_grad_norm-ClipGradNorm/SumExt-op15,Default/clip_grad_norm-ClipGradNorm/SumExt-op15,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15156,15161,0x12c18000c000,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14319,375_Default/clip_grad_norm-ClipGradNorm/Square-op16,Default/clip_grad_norm-ClipGradNorm/Square-op16,kernel_graph0,{15159},15159,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15158,15216,0x12c18000b600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14319,376_Default/clip_grad_norm-ClipGradNorm/SumExt-op16,Default/clip_grad_norm-ClipGradNorm/SumExt-op16,kernel_graph0,{15162-15198},15198,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15159,15162,0x12c18000b800,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14319,376_Default/clip_grad_norm-ClipGradNorm/SumExt-op16,Default/clip_grad_norm-ClipGradNorm/SumExt-op16,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15164,15169,0x12c18000c000,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14323,378_Default/clip_grad_norm-ClipGradNorm/Square-op17,Default/clip_grad_norm-ClipGradNorm/Square-op17,kernel_graph0,{15167},15167,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15166,15217,0x12c18000b800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14323,379_Default/clip_grad_norm-ClipGradNorm/SumExt-op17,Default/clip_grad_norm-ClipGradNorm/SumExt-op17,kernel_graph0,{15170-15198},15198,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15167,15170,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14323,379_Default/clip_grad_norm-ClipGradNorm/SumExt-op17,Default/clip_grad_norm-ClipGradNorm/SumExt-op17,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15172,15177,0x12c18000c000,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14325,381_Default/clip_grad_norm-ClipGradNorm/Square-op18,Default/clip_grad_norm-ClipGradNorm/Square-op18,kernel_graph0,{15175},15175,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15174,15218,0x12c18000ba00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14326,382_Default/clip_grad_norm-ClipGradNorm/SumExt-op18,Default/clip_grad_norm-ClipGradNorm/SumExt-op18,kernel_graph0,{15178-15198},15198,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15175,15178,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14326,382_Default/clip_grad_norm-ClipGradNorm/SumExt-op18,Default/clip_grad_norm-ClipGradNorm/SumExt-op18,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15180,15185,0x12c18000c000,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14327,384_Default/clip_grad_norm-ClipGradNorm/Square-op19,Default/clip_grad_norm-ClipGradNorm/Square-op19,kernel_graph0,{15183},15183,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15182,15219,0x12c18000bc00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14329,385_Default/clip_grad_norm-ClipGradNorm/SumExt-op19,Default/clip_grad_norm-ClipGradNorm/SumExt-op19,kernel_graph0,{15186-15198},15198,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15183,15186,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14329,385_Default/clip_grad_norm-ClipGradNorm/SumExt-op19,Default/clip_grad_norm-ClipGradNorm/SumExt-op19,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15188,18553,0x12c18000c000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14330,387_Default/AllFinite-op0,Default/AllFinite-op0,kernel_graph0,{15224-15227-15229},15229,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15190,15195,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14332,388_Default/clip_grad_norm-ClipGradNorm/Square-op20,Default/clip_grad_norm-ClipGradNorm/Square-op20,kernel_graph0,{15193},15193,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15192,15220,0x12c18000c200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14333,389_Default/clip_grad_norm-ClipGradNorm/SumExt-op20,Default/clip_grad_norm-ClipGradNorm/SumExt-op20,kernel_graph0,{15196-15198},15198,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15193,15196,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14333,389_Default/clip_grad_norm-ClipGradNorm/SumExt-op20,Default/clip_grad_norm-ClipGradNorm/SumExt-op20,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15198,15223,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14336,391_Default/clip_grad_norm-ClipGradNorm/AddN-op0,Default/clip_grad_norm-ClipGradNorm/AddN-op0,kernel_graph0,{15221},15221,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15221,18550,0x12c18000c200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14337,392_Default/clip_grad_norm-ClipGradNorm/Sqrt-op0,Default/clip_grad_norm-ClipGradNorm/Sqrt-op0,kernel_graph0,{15231-15233},15233,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15224,15254,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14337,393_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Select-op1,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Select-op1,kernel_graph0,{15245-15250},15250,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15227,15371,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14338,394_Default/LogicalNot-op0,Default/LogicalNot-op0,kernel_graph0,{15247},15247,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15229,15257,0x12c180009200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14339,395_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/LogicalOr-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/LogicalOr-op0,kernel_graph0,{15255},15255,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15231,15235,0x12c180009a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14340,396_Default/clip_grad_norm-ClipGradNorm/GreaterEqual-op0,Default/clip_grad_norm-ClipGradNorm/GreaterEqual-op0,kernel_graph0,{15233},15233,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15233,15238,0x12c180009c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14342,397_Default/clip_grad_norm-ClipGradNorm/Select-op0,Default/clip_grad_norm-ClipGradNorm/Select-op0,kernel_graph0,{15236},15236,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15236,15241,0x12c180009a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14343,398_Default/learning_rate-CosineWithWarmUpLR/Add-op0,Default/learning_rate-CosineWithWarmUpLR/Add-op0,kernel_graph0,{15239},15239,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15239,15244,0x12c180009c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14344,399_Default/clip_grad_norm-ClipGradNorm/Div-op0,Default/clip_grad_norm-ClipGradNorm/Div-op0,kernel_graph0,{15242},15242,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15242,15323,0x12c180009a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14345,400_Default/clip_grad_norm-ClipGradNorm/Minimum-op0,Default/clip_grad_norm-ClipGradNorm/Minimum-op0,kernel_graph0,{15260-15263-15266-15269-15272-15275-15278-15281-15284-15287-15290-15293-15296-15299-15302-15305-15308-15311-15314-15317-15320},15320,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15245,15253,0x12c180009c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14345,401_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Mul-op1,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Mul-op1,kernel_graph0,{15250},15250,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15247,15252,0x12c180009e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14346,402_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/LogicalAnd-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/LogicalAnd-op0,kernel_graph0,{15250},15250,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15250,15329,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14347,403_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Select-op2,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Select-op2,kernel_graph0,{15327},15327,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15255,15325,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14347,404_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Select-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Select-op0,kernel_graph0,{15323},15323,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15258,15327,0x12c180009200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14348,405_Default/learning_rate-CosineWithWarmUpLR/Add-op1,Default/learning_rate-CosineWithWarmUpLR/Add-op1,kernel_graph0,{15325},15325,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15260,15530,0x12c1a6d35c00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14348,406_Default/clip_grad_norm-ClipGradNorm/Mul-op0,Default/clip_grad_norm-ClipGradNorm/Mul-op0,kernel_graph0,{15526-15528},15528,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15263,15520,0x12c209e0ea00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14348,407_Default/clip_grad_norm-ClipGradNorm/Mul-op1,Default/clip_grad_norm-ClipGradNorm/Mul-op1,kernel_graph0,{15516-15518},15518,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15266,15512,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14349,408_Default/clip_grad_norm-ClipGradNorm/Mul-op2,Default/clip_grad_norm-ClipGradNorm/Mul-op2,kernel_graph0,{15508-15510},15510,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15269,15504,0x12c1e9c0de00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14349,409_Default/clip_grad_norm-ClipGradNorm/Mul-op3,Default/clip_grad_norm-ClipGradNorm/Mul-op3,kernel_graph0,{15500-15502},15502,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15272,15496,0x12c1f5c12600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14350,410_Default/clip_grad_norm-ClipGradNorm/Mul-op4,Default/clip_grad_norm-ClipGradNorm/Mul-op4,kernel_graph0,{15492-15494},15494,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15275,15488,0x12c1c6135e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14350,411_Default/clip_grad_norm-ClipGradNorm/Mul-op5,Default/clip_grad_norm-ClipGradNorm/Mul-op5,kernel_graph0,{15484-15486},15486,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15278,15480,0x12c1d0d36000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14353,412_Default/clip_grad_norm-ClipGradNorm/Mul-op6,Default/clip_grad_norm-ClipGradNorm/Mul-op6,kernel_graph0,{15476-15478},15478,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15281,15472,0x12c20de0ec00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14354,413_Default/clip_grad_norm-ClipGradNorm/Mul-op7,Default/clip_grad_norm-ClipGradNorm/Mul-op7,kernel_graph0,{15468-15470},15470,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15284,15464,0x12c23d610200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14355,414_Default/clip_grad_norm-ClipGradNorm/Mul-op8,Default/clip_grad_norm-ClipGradNorm/Mul-op8,kernel_graph0,{15460-15462},15462,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15287,15456,0x12c1db936200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14355,415_Default/clip_grad_norm-ClipGradNorm/Mul-op9,Default/clip_grad_norm-ClipGradNorm/Mul-op9,kernel_graph0,{15452-15454},15454,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15290,15448,0x12c1edc0e000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14355,416_Default/clip_grad_norm-ClipGradNorm/Mul-op10,Default/clip_grad_norm-ClipGradNorm/Mul-op10,kernel_graph0,{15444-15446},15446,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15293,15440,0x12c1df936400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14355,417_Default/clip_grad_norm-ClipGradNorm/Mul-op11,Default/clip_grad_norm-ClipGradNorm/Mul-op11,kernel_graph0,{15436-15438},15438,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15296,15432,0x12c224a0f600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14358,418_Default/clip_grad_norm-ClipGradNorm/Mul-op12,Default/clip_grad_norm-ClipGradNorm/Mul-op12,kernel_graph0,{15428-15430},15430,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15299,15424,0x12c22f60f800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14359,419_Default/clip_grad_norm-ClipGradNorm/Mul-op13,Default/clip_grad_norm-ClipGradNorm/Mul-op13,kernel_graph0,{15420-15422},15422,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15302,15416,0x12c218a0ee00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14359,420_Default/clip_grad_norm-ClipGradNorm/Mul-op14,Default/clip_grad_norm-ClipGradNorm/Mul-op14,kernel_graph0,{15412-15414},15414,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15305,15408,0x12c180018600,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14361,421_Default/clip_grad_norm-ClipGradNorm/Mul-op15,Default/clip_grad_norm-ClipGradNorm/Mul-op15,kernel_graph0,{15404-15406},15406,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15308,15400,0x12c22360f000,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14362,422_Default/clip_grad_norm-ClipGradNorm/Mul-op16,Default/clip_grad_norm-ClipGradNorm/Mul-op16,kernel_graph0,{15396-15398},15398,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15311,15392,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14363,423_Default/clip_grad_norm-ClipGradNorm/Mul-op17,Default/clip_grad_norm-ClipGradNorm/Mul-op17,kernel_graph0,{15388-15390},15390,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15314,15384,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14364,424_Default/clip_grad_norm-ClipGradNorm/Mul-op18,Default/clip_grad_norm-ClipGradNorm/Mul-op18,kernel_graph0,{15380-15382},15382,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15317,15376,0x12c180014400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14364,425_Default/clip_grad_norm-ClipGradNorm/Mul-op19,Default/clip_grad_norm-ClipGradNorm/Mul-op19,kernel_graph0,{15372-15374},15374,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15320,15538,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14366,426_Default/clip_grad_norm-ClipGradNorm/Mul-op20,Default/clip_grad_norm-ClipGradNorm/Mul-op20,kernel_graph0,{15534-15536},15536,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15330,15339,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14368,430_Default/learning_rate-CosineWithWarmUpLR/Cast-op0,Default/learning_rate-CosineWithWarmUpLR/Cast-op0,kernel_graph0,{15332-15335-15337},15337,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15332,15334,0x12c180009200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,14369,431_Default/learning_rate-CosineWithWarmUpLR/GreaterEqual-op0,Default/learning_rate-CosineWithWarmUpLR/GreaterEqual-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15335,15340,0x12c180009200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15334,435_Default/learning_rate-CosineWithWarmUpLR/Greater-op0,Default/learning_rate-CosineWithWarmUpLR/Greater-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15337,15343,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15335,436_Default/TensorMove-op0,Default/TensorMove-op0,kernel_graph0,{15341},15341,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15341,15346,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15340,440_Default/learning_rate-CosineWithWarmUpLR/Sub-op1,Default/learning_rate-CosineWithWarmUpLR/Sub-op1,kernel_graph0,{15344},15344,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15344,15349,0x12c180009200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15341,441_Default/learning_rate-CosineWithWarmUpLR/Div-op1,Default/learning_rate-CosineWithWarmUpLR/Div-op1,kernel_graph0,{15347},15347,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15347,15352,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15342,442_Default/learning_rate-CosineWithWarmUpLR/Mul-op1,Default/learning_rate-CosineWithWarmUpLR/Mul-op1,kernel_graph0,{15350},15350,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15350,15355,0x12c180009200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15342,443_Default/learning_rate-CosineWithWarmUpLR/Cos-op0,Default/learning_rate-CosineWithWarmUpLR/Cos-op0,kernel_graph0,{15353},15353,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15353,15358,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15344,444_Default/learning_rate-CosineWithWarmUpLR/Add-op3,Default/learning_rate-CosineWithWarmUpLR/Add-op3,kernel_graph0,{15356},15356,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15356,15361,0x12c180009200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15344,445_Default/learning_rate-CosineWithWarmUpLR/Mul-op2,Default/learning_rate-CosineWithWarmUpLR/Mul-op2,kernel_graph0,{15359},15359,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15359,15364,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15345,446_Default/learning_rate-CosineWithWarmUpLR/Maximum-op0,Default/learning_rate-CosineWithWarmUpLR/Maximum-op0,kernel_graph0,{15362},15362,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15362,15367,0x12c180009200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15346,447_Default/learning_rate-CosineWithWarmUpLR/Mul-op3,Default/learning_rate-CosineWithWarmUpLR/Mul-op3,kernel_graph0,{15365},15365,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15365,15370,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15347,448_Default/learning_rate-CosineWithWarmUpLR/Add-op4,Default/learning_rate-CosineWithWarmUpLR/Add-op4,kernel_graph0,{15368},15368,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15368,18551,0x12c180009200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15350,451_Default/Reshape-op0,Default/Reshape-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15372,16201,0x12c1f1c0e200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15371,453_Default/optimizer-FP32StateAdamWeightDecay/Mul-op96,Default/optimizer-FP32StateAdamWeightDecay/Mul-op96,kernel_graph0,{16198},16198,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15374,15379,0x12c223613200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15372,454_Default/optimizer-FP32StateAdamWeightDecay/Square-op19,Default/optimizer-FP32StateAdamWeightDecay/Square-op19,kernel_graph0,{15377},15377,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15377,16197,0x12c180010600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15373,455_Default/optimizer-FP32StateAdamWeightDecay/Mul-op98,Default/optimizer-FP32StateAdamWeightDecay/Mul-op98,kernel_graph0,{16194},16194,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15380,16165,0x12c223613200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15373,456_Default/optimizer-FP32StateAdamWeightDecay/Mul-op91,Default/optimizer-FP32StateAdamWeightDecay/Mul-op91,kernel_graph0,{16162},16162,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15382,15387,0x12c223617400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15374,457_Default/optimizer-FP32StateAdamWeightDecay/Square-op18,Default/optimizer-FP32StateAdamWeightDecay/Square-op18,kernel_graph0,{15385},15385,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15385,16169,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15375,458_Default/optimizer-FP32StateAdamWeightDecay/Mul-op93,Default/optimizer-FP32StateAdamWeightDecay/Mul-op93,kernel_graph0,{16166},16166,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15388,16133,0x12c223617400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15375,459_Default/optimizer-FP32StateAdamWeightDecay/Mul-op86,Default/optimizer-FP32StateAdamWeightDecay/Mul-op86,kernel_graph0,{16130},16130,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15390,15395,0x12c22361b600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15377,460_Default/optimizer-FP32StateAdamWeightDecay/Square-op17,Default/optimizer-FP32StateAdamWeightDecay/Square-op17,kernel_graph0,{15393},15393,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15393,16137,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15378,461_Default/optimizer-FP32StateAdamWeightDecay/Mul-op88,Default/optimizer-FP32StateAdamWeightDecay/Mul-op88,kernel_graph0,{16134},16134,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15396,16101,0x12c22361b600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15378,462_Default/optimizer-FP32StateAdamWeightDecay/Mul-op81,Default/optimizer-FP32StateAdamWeightDecay/Mul-op81,kernel_graph0,{16098},16098,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15398,15403,0x12c22361f800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15380,463_Default/optimizer-FP32StateAdamWeightDecay/Square-op16,Default/optimizer-FP32StateAdamWeightDecay/Square-op16,kernel_graph0,{15401},15401,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15401,16105,0x12c22360f000,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15380,464_Default/optimizer-FP32StateAdamWeightDecay/Mul-op83,Default/optimizer-FP32StateAdamWeightDecay/Mul-op83,kernel_graph0,{16102},16102,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15404,16069,0x12c241610400,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15381,465_Default/optimizer-FP32StateAdamWeightDecay/Mul-op76,Default/optimizer-FP32StateAdamWeightDecay/Mul-op76,kernel_graph0,{16066},16066,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15406,15411,0x12c260a10600,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15382,466_Default/optimizer-FP32StateAdamWeightDecay/Square-op15,Default/optimizer-FP32StateAdamWeightDecay/Square-op15,kernel_graph0,{15409},15409,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15409,16073,0x12c180014800,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15382,467_Default/optimizer-FP32StateAdamWeightDecay/Mul-op78,Default/optimizer-FP32StateAdamWeightDecay/Mul-op78,kernel_graph0,{16070},16070,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15412,16037,0x12c1f9c12800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15383,468_Default/optimizer-FP32StateAdamWeightDecay/Mul-op71,Default/optimizer-FP32StateAdamWeightDecay/Mul-op71,kernel_graph0,{16034},16034,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15414,15419,0x12c260a10600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15384,469_Default/optimizer-FP32StateAdamWeightDecay/Square-op14,Default/optimizer-FP32StateAdamWeightDecay/Square-op14,kernel_graph0,{15417},15417,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15417,16041,0x12c218a0ee00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15385,470_Default/optimizer-FP32StateAdamWeightDecay/Mul-op73,Default/optimizer-FP32StateAdamWeightDecay/Mul-op73,kernel_graph0,{16038},16038,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15420,16005,0x12c260a10600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15386,471_Default/optimizer-FP32StateAdamWeightDecay/Mul-op66,Default/optimizer-FP32StateAdamWeightDecay/Mul-op66,kernel_graph0,{16002},16002,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15422,15427,0x12c26b610800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15387,472_Default/optimizer-FP32StateAdamWeightDecay/Square-op13,Default/optimizer-FP32StateAdamWeightDecay/Square-op13,kernel_graph0,{15425},15425,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15425,16009,0x12c22f60f800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15388,473_Default/optimizer-FP32StateAdamWeightDecay/Mul-op68,Default/optimizer-FP32StateAdamWeightDecay/Mul-op68,kernel_graph0,{16006},16006,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15428,15973,0x12c26b610800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15389,474_Default/optimizer-FP32StateAdamWeightDecay/Mul-op61,Default/optimizer-FP32StateAdamWeightDecay/Mul-op61,kernel_graph0,{15970},15970,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15430,15435,0x12c276210a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15390,475_Default/optimizer-FP32StateAdamWeightDecay/Square-op12,Default/optimizer-FP32StateAdamWeightDecay/Square-op12,kernel_graph0,{15433},15433,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15433,15977,0x12c22361f800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15391,476_Default/optimizer-FP32StateAdamWeightDecay/Mul-op63,Default/optimizer-FP32StateAdamWeightDecay/Mul-op63,kernel_graph0,{15974},15974,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15436,15941,0x12c204812a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15392,477_Default/optimizer-FP32StateAdamWeightDecay/Mul-op56,Default/optimizer-FP32StateAdamWeightDecay/Mul-op56,kernel_graph0,{15938},15938,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15438,15443,0x12c1e3936600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15393,478_Default/optimizer-FP32StateAdamWeightDecay/Square-op11,Default/optimizer-FP32StateAdamWeightDecay/Square-op11,kernel_graph0,{15441},15441,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15441,15945,0x12c1df936400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15394,479_Default/optimizer-FP32StateAdamWeightDecay/Mul-op58,Default/optimizer-FP32StateAdamWeightDecay/Mul-op58,kernel_graph0,{15942},15942,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15444,15909,0x12c1e3936600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15396,480_Default/optimizer-FP32StateAdamWeightDecay/Mul-op51,Default/optimizer-FP32StateAdamWeightDecay/Mul-op51,kernel_graph0,{15906},15906,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15446,15451,0x12c19f414a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15396,481_Default/optimizer-FP32StateAdamWeightDecay/Square-op10,Default/optimizer-FP32StateAdamWeightDecay/Square-op10,kernel_graph0,{15449},15449,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15449,15913,0x12c1edc0e000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15397,482_Default/optimizer-FP32StateAdamWeightDecay/Mul-op53,Default/optimizer-FP32StateAdamWeightDecay/Mul-op53,kernel_graph0,{15910},15910,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15452,15877,0x12c19f414a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15397,483_Default/optimizer-FP32StateAdamWeightDecay/Mul-op46,Default/optimizer-FP32StateAdamWeightDecay/Mul-op46,kernel_graph0,{15874},15874,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15454,15459,0x12c276210a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15398,484_Default/optimizer-FP32StateAdamWeightDecay/Square-op9,Default/optimizer-FP32StateAdamWeightDecay/Square-op9,kernel_graph0,{15457},15457,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15457,15881,0x12c1db936200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15399,485_Default/optimizer-FP32StateAdamWeightDecay/Mul-op48,Default/optimizer-FP32StateAdamWeightDecay/Mul-op48,kernel_graph0,{15878},15878,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15460,15845,0x12c276210a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15400,486_Default/optimizer-FP32StateAdamWeightDecay/Mul-op41,Default/optimizer-FP32StateAdamWeightDecay/Mul-op41,kernel_graph0,{15842},15842,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15462,15467,0x12c27a210c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15401,487_Default/optimizer-FP32StateAdamWeightDecay/Square-op8,Default/optimizer-FP32StateAdamWeightDecay/Square-op8,kernel_graph0,{15465},15465,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15465,15849,0x12c23a20fa00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15402,488_Default/optimizer-FP32StateAdamWeightDecay/Mul-op43,Default/optimizer-FP32StateAdamWeightDecay/Mul-op43,kernel_graph0,{15846},15846,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15468,15813,0x12c27a210c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15403,489_Default/optimizer-FP32StateAdamWeightDecay/Mul-op36,Default/optimizer-FP32StateAdamWeightDecay/Mul-op36,kernel_graph0,{15810},15810,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15470,15475,0x12c284e10e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15404,490_Default/optimizer-FP32StateAdamWeightDecay/Square-op7,Default/optimizer-FP32StateAdamWeightDecay/Square-op7,kernel_graph0,{15473},15473,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15473,15817,0x12c20de0ec00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15405,491_Default/optimizer-FP32StateAdamWeightDecay/Mul-op38,Default/optimizer-FP32StateAdamWeightDecay/Mul-op38,kernel_graph0,{15814},15814,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15476,15781,0x12c284e10e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15406,492_Default/optimizer-FP32StateAdamWeightDecay/Mul-op31,Default/optimizer-FP32StateAdamWeightDecay/Mul-op31,kernel_graph0,{15778},15778,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15478,15483,0x12c28fa11000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15406,493_Default/optimizer-FP32StateAdamWeightDecay/Square-op6,Default/optimizer-FP32StateAdamWeightDecay/Square-op6,kernel_graph0,{15481},15481,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15481,15785,0x12c1d0d36000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15407,494_Default/optimizer-FP32StateAdamWeightDecay/Mul-op33,Default/optimizer-FP32StateAdamWeightDecay/Mul-op33,kernel_graph0,{15782},15782,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15484,15749,0x12c28fa11000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15409,495_Default/optimizer-FP32StateAdamWeightDecay/Mul-op26,Default/optimizer-FP32StateAdamWeightDecay/Mul-op26,kernel_graph0,{15746},15746,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15486,15491,0x12c29a611200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15409,496_Default/optimizer-FP32StateAdamWeightDecay/Square-op5,Default/optimizer-FP32StateAdamWeightDecay/Square-op5,kernel_graph0,{15489},15489,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15489,15753,0x12c1c6135e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15410,497_Default/optimizer-FP32StateAdamWeightDecay/Mul-op28,Default/optimizer-FP32StateAdamWeightDecay/Mul-op28,kernel_graph0,{15750},15750,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15492,15717,0x12c29a611200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15411,498_Default/optimizer-FP32StateAdamWeightDecay/Mul-op21,Default/optimizer-FP32StateAdamWeightDecay/Mul-op21,kernel_graph0,{15714},15714,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15494,15499,0x12c29e611400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15412,499_Default/optimizer-FP32StateAdamWeightDecay/Square-op4,Default/optimizer-FP32StateAdamWeightDecay/Square-op4,kernel_graph0,{15497},15497,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15497,15721,0x12c1f5c12600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15413,500_Default/optimizer-FP32StateAdamWeightDecay/Mul-op23,Default/optimizer-FP32StateAdamWeightDecay/Mul-op23,kernel_graph0,{15718},15718,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15500,15685,0x12c29e611400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15413,501_Default/optimizer-FP32StateAdamWeightDecay/Mul-op16,Default/optimizer-FP32StateAdamWeightDecay/Mul-op16,kernel_graph0,{15682},15682,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15502,15507,0x12c2a2611600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15414,502_Default/optimizer-FP32StateAdamWeightDecay/Square-op3,Default/optimizer-FP32StateAdamWeightDecay/Square-op3,kernel_graph0,{15505},15505,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15505,15689,0x12c1e7936800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15415,503_Default/optimizer-FP32StateAdamWeightDecay/Mul-op18,Default/optimizer-FP32StateAdamWeightDecay/Mul-op18,kernel_graph0,{15686},15686,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15508,15653,0x12c2a2611600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15415,504_Default/optimizer-FP32StateAdamWeightDecay/Mul-op11,Default/optimizer-FP32StateAdamWeightDecay/Mul-op11,kernel_graph0,{15650},15650,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15510,15515,0x12c2a6611800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15417,505_Default/optimizer-FP32StateAdamWeightDecay/Square-op2,Default/optimizer-FP32StateAdamWeightDecay/Square-op2,kernel_graph0,{15513},15513,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15513,15657,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15418,506_Default/optimizer-FP32StateAdamWeightDecay/Mul-op13,Default/optimizer-FP32StateAdamWeightDecay/Mul-op13,kernel_graph0,{15654},15654,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15516,15621,0x12c2a6611800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15418,507_Default/optimizer-FP32StateAdamWeightDecay/Mul-op6,Default/optimizer-FP32StateAdamWeightDecay/Mul-op6,kernel_graph0,{15618},15618,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15518,15523,0x12c2aa611a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15420,508_Default/optimizer-FP32StateAdamWeightDecay/Square-op1,Default/optimizer-FP32StateAdamWeightDecay/Square-op1,kernel_graph0,{15521},15521,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15521,15625,0x12c208812c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15421,509_Default/optimizer-FP32StateAdamWeightDecay/Mul-op8,Default/optimizer-FP32StateAdamWeightDecay/Mul-op8,kernel_graph0,{15622},15622,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15524,15549,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15421,510_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Cast-op0,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Cast-op0,kernel_graph0,{15542-15545-15547},15547,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15526,15589,0x12c2aa611a00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15422,511_Default/optimizer-FP32StateAdamWeightDecay/Mul-op1,Default/optimizer-FP32StateAdamWeightDecay/Mul-op1,kernel_graph0,{15586},15586,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15528,15533,0x12c2c9a11c00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15423,512_Default/optimizer-FP32StateAdamWeightDecay/Square-op0,Default/optimizer-FP32StateAdamWeightDecay/Square-op0,kernel_graph0,{15531},15531,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15531,15593,0x12c1a3414c00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15423,513_Default/optimizer-FP32StateAdamWeightDecay/Mul-op3,Default/optimizer-FP32StateAdamWeightDecay/Mul-op3,kernel_graph0,{15590},15590,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15534,16233,0x12c22e21fa00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15425,514_Default/optimizer-FP32StateAdamWeightDecay/Mul-op101,Default/optimizer-FP32StateAdamWeightDecay/Mul-op101,kernel_graph0,{16230},16230,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15536,15541,0x12c22e223c00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15426,515_Default/optimizer-FP32StateAdamWeightDecay/Square-op20,Default/optimizer-FP32StateAdamWeightDecay/Square-op20,kernel_graph0,{15539},15539,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15539,16229,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15426,516_Default/optimizer-FP32StateAdamWeightDecay/Mul-op103,Default/optimizer-FP32StateAdamWeightDecay/Mul-op103,kernel_graph0,{16226},16226,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15542,15544,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15428,518_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/GreaterEqual-op0,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/GreaterEqual-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15545,15550,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15544,522_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Greater-op0,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Greater-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15547,15553,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15545,523_Default/TensorMove-op1,Default/TensorMove-op1,kernel_graph0,{15551},15551,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15551,15556,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15550,527_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Sub-op1,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Sub-op1,kernel_graph0,{15554},15554,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15554,15559,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15551,528_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Div-op1,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Div-op1,kernel_graph0,{15557},15557,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15557,15562,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15552,529_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Mul-op1,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Mul-op1,kernel_graph0,{15560},15560,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15560,15565,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15552,530_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Cos-op0,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Cos-op0,kernel_graph0,{15563},15563,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15563,15568,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15554,531_Default/learning_rate-CosineWithWarmUpLR/Add-op6,Default/learning_rate-CosineWithWarmUpLR/Add-op6,kernel_graph0,{15566},15566,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15566,15571,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15555,532_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Mul-op2,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Mul-op2,kernel_graph0,{15569},15569,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15569,15574,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15555,533_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Maximum-op0,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Maximum-op0,kernel_graph0,{15572},15572,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15572,15577,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15557,534_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Mul-op3,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Mul-op3,kernel_graph0,{15575},15575,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15575,15580,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15558,535_Default/learning_rate-CosineWithWarmUpLR/Add-op7,Default/learning_rate-CosineWithWarmUpLR/Add-op7,kernel_graph0,{15578},15578,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15578,16244,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15561,538_Default/optimizer-FP32StateAdamWeightDecay/Reshape-op0,Default/optimizer-FP32StateAdamWeightDecay/Reshape-op0,kernel_graph0,{15602-15634-15666-15698-15730-15762-15794-15826-15858-15890-15922-15954-15986-16018-16050-16082-16114-16146-16178-16210-16242},16242,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15582,15588,0x12c2c9a11c00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15563,540_Default/optimizer-FP32StateAdamWeightDecay/Mul-op0,Default/optimizer-FP32StateAdamWeightDecay/Mul-op0,kernel_graph0,{15586},15586,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15584,15592,0x12c2e8e11e00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15564,541_Default/optimizer-FP32StateAdamWeightDecay/Mul-op2,Default/optimizer-FP32StateAdamWeightDecay/Mul-op2,kernel_graph0,{15590},15590,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15586,15611,0x12c308212000,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15564,542_Default/learning_rate-CosineWithWarmUpLR/Add-op8,Default/learning_rate-CosineWithWarmUpLR/Add-op8,kernel_graph0,{15599-15609},15609,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15590,15609,0x12c2aa611a00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15566,543_Default/learning_rate-CosineWithWarmUpLR/Add-op9,Default/learning_rate-CosineWithWarmUpLR/Add-op9,kernel_graph0,{15594-15607},15607,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15594,15598,0x12c1a3414c00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15567,544_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op0,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op0,kernel_graph0,{15596},15596,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15596,15601,0x12c2c9a11c00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15567,545_Default/learning_rate-CosineWithWarmUpLR/Add-op10,Default/learning_rate-CosineWithWarmUpLR/Add-op10,kernel_graph0,{15599},15599,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15599,15604,0x12c2e8e11e00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15569,546_Default/optimizer-FP32StateAdamWeightDecay/Div-op0,Default/optimizer-FP32StateAdamWeightDecay/Div-op0,kernel_graph0,{15602},15602,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15602,15607,0x12c2c9a11c00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15570,547_Default/optimizer-FP32StateAdamWeightDecay/Mul-op4,Default/optimizer-FP32StateAdamWeightDecay/Mul-op4,kernel_graph0,{15605},15605,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15605,15613,0x12c2e8e11e00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15570,548_Default/optimizer-FP32StateAdamWeightDecay/Sub-op0,Default/optimizer-FP32StateAdamWeightDecay/Sub-op0,kernel_graph0,{15611},15611,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15614,15620,0x12c1a3414c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15575,552_Default/optimizer-FP32StateAdamWeightDecay/Mul-op5,Default/optimizer-FP32StateAdamWeightDecay/Mul-op5,kernel_graph0,{15618},15618,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15616,15624,0x12c1a7414e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15576,553_Default/optimizer-FP32StateAdamWeightDecay/Mul-op7,Default/optimizer-FP32StateAdamWeightDecay/Mul-op7,kernel_graph0,{15622},15622,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15618,15643,0x12c1ab415000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15576,554_Default/learning_rate-CosineWithWarmUpLR/Add-op11,Default/learning_rate-CosineWithWarmUpLR/Add-op11,kernel_graph0,{15631-15641},15641,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15622,15641,0x12c1a3414c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15578,555_Default/learning_rate-CosineWithWarmUpLR/Add-op12,Default/learning_rate-CosineWithWarmUpLR/Add-op12,kernel_graph0,{15626-15639},15639,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15626,15630,0x12c1a7414e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15579,556_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op1,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op1,kernel_graph0,{15628},15628,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15628,15633,0x12c208812c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15580,557_Default/learning_rate-CosineWithWarmUpLR/Add-op13,Default/learning_rate-CosineWithWarmUpLR/Add-op13,kernel_graph0,{15631},15631,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15631,15636,0x12c1a7414e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15580,558_Default/optimizer-FP32StateAdamWeightDecay/Div-op1,Default/optimizer-FP32StateAdamWeightDecay/Div-op1,kernel_graph0,{15634},15634,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15634,15639,0x12c208812c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15581,559_Default/optimizer-FP32StateAdamWeightDecay/Mul-op9,Default/optimizer-FP32StateAdamWeightDecay/Mul-op9,kernel_graph0,{15637},15637,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15637,15645,0x12c1a7414e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15581,560_Default/optimizer-FP32StateAdamWeightDecay/Sub-op1,Default/optimizer-FP32StateAdamWeightDecay/Sub-op1,kernel_graph0,{15643},15643,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15646,15652,0x12c208812c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15583,564_Default/optimizer-FP32StateAdamWeightDecay/Mul-op10,Default/optimizer-FP32StateAdamWeightDecay/Mul-op10,kernel_graph0,{15650},15650,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15648,15656,0x12c1a3414c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15583,565_Default/optimizer-FP32StateAdamWeightDecay/Mul-op12,Default/optimizer-FP32StateAdamWeightDecay/Mul-op12,kernel_graph0,{15654},15654,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15650,15675,0x12c1a7414e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15584,566_Default/learning_rate-CosineWithWarmUpLR/Add-op14,Default/learning_rate-CosineWithWarmUpLR/Add-op14,kernel_graph0,{15663-15673},15673,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15654,15673,0x12c208812c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15585,567_Default/learning_rate-CosineWithWarmUpLR/Add-op15,Default/learning_rate-CosineWithWarmUpLR/Add-op15,kernel_graph0,{15658-15671},15671,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15658,15662,0x12c1a3414c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15585,568_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op2,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op2,kernel_graph0,{15660},15660,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15660,15665,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15586,569_Default/learning_rate-CosineWithWarmUpLR/Add-op16,Default/learning_rate-CosineWithWarmUpLR/Add-op16,kernel_graph0,{15663},15663,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15663,15668,0x12c1a3414c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15587,570_Default/optimizer-FP32StateAdamWeightDecay/Div-op2,Default/optimizer-FP32StateAdamWeightDecay/Div-op2,kernel_graph0,{15666},15666,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15666,15671,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15587,571_Default/optimizer-FP32StateAdamWeightDecay/Mul-op14,Default/optimizer-FP32StateAdamWeightDecay/Mul-op14,kernel_graph0,{15669},15669,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15669,15677,0x12c1a3414c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15590,572_Default/optimizer-FP32StateAdamWeightDecay/Sub-op2,Default/optimizer-FP32StateAdamWeightDecay/Sub-op2,kernel_graph0,{15675},15675,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15678,15684,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15595,576_Default/optimizer-FP32StateAdamWeightDecay/Mul-op15,Default/optimizer-FP32StateAdamWeightDecay/Mul-op15,kernel_graph0,{15682},15682,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15680,15688,0x12c208812c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15595,577_Default/optimizer-FP32StateAdamWeightDecay/Mul-op17,Default/optimizer-FP32StateAdamWeightDecay/Mul-op17,kernel_graph0,{15686},15686,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15682,15707,0x12c1a3414c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15596,578_Default/learning_rate-CosineWithWarmUpLR/Add-op17,Default/learning_rate-CosineWithWarmUpLR/Add-op17,kernel_graph0,{15695-15705},15705,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15686,15705,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15597,579_Default/learning_rate-CosineWithWarmUpLR/Add-op18,Default/learning_rate-CosineWithWarmUpLR/Add-op18,kernel_graph0,{15690-15703},15703,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15690,15694,0x12c208812c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15599,580_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op3,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op3,kernel_graph0,{15692},15692,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15692,15697,0x12c1e7936800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15599,581_Default/learning_rate-CosineWithWarmUpLR/Add-op19,Default/learning_rate-CosineWithWarmUpLR/Add-op19,kernel_graph0,{15695},15695,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15695,15700,0x12c208812c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15600,582_Default/optimizer-FP32StateAdamWeightDecay/Div-op3,Default/optimizer-FP32StateAdamWeightDecay/Div-op3,kernel_graph0,{15698},15698,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15698,15703,0x12c1e7936800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15601,583_Default/optimizer-FP32StateAdamWeightDecay/Mul-op19,Default/optimizer-FP32StateAdamWeightDecay/Mul-op19,kernel_graph0,{15701},15701,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15701,15709,0x12c208812c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15602,584_Default/optimizer-FP32StateAdamWeightDecay/Sub-op3,Default/optimizer-FP32StateAdamWeightDecay/Sub-op3,kernel_graph0,{15707},15707,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15710,15716,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15606,588_Default/optimizer-FP32StateAdamWeightDecay/Mul-op20,Default/optimizer-FP32StateAdamWeightDecay/Mul-op20,kernel_graph0,{15714},15714,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15712,15720,0x12c208812c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15607,589_Default/optimizer-FP32StateAdamWeightDecay/Mul-op22,Default/optimizer-FP32StateAdamWeightDecay/Mul-op22,kernel_graph0,{15718},15718,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15714,15739,0x12c1e7936800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15608,590_Default/learning_rate-CosineWithWarmUpLR/Add-op20,Default/learning_rate-CosineWithWarmUpLR/Add-op20,kernel_graph0,{15727-15737},15737,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15718,15737,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15608,591_Default/learning_rate-CosineWithWarmUpLR/Add-op21,Default/learning_rate-CosineWithWarmUpLR/Add-op21,kernel_graph0,{15722-15735},15735,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15722,15726,0x12c1f5c12600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15608,592_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op4,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op4,kernel_graph0,{15724},15724,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15724,15729,0x12c208812c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15608,593_Default/learning_rate-CosineWithWarmUpLR/Add-op22,Default/learning_rate-CosineWithWarmUpLR/Add-op22,kernel_graph0,{15727},15727,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15727,15732,0x12c1f5c12600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15609,594_Default/optimizer-FP32StateAdamWeightDecay/Div-op4,Default/optimizer-FP32StateAdamWeightDecay/Div-op4,kernel_graph0,{15730},15730,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15730,15735,0x12c208812c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15609,595_Default/optimizer-FP32StateAdamWeightDecay/Mul-op24,Default/optimizer-FP32StateAdamWeightDecay/Mul-op24,kernel_graph0,{15733},15733,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15733,15741,0x12c1f5c12600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15610,596_Default/optimizer-FP32StateAdamWeightDecay/Sub-op4,Default/optimizer-FP32StateAdamWeightDecay/Sub-op4,kernel_graph0,{15739},15739,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15742,15748,0x12c1a3414c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15612,600_Default/optimizer-FP32StateAdamWeightDecay/Mul-op25,Default/optimizer-FP32StateAdamWeightDecay/Mul-op25,kernel_graph0,{15746},15746,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15744,15752,0x12c1ae014e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15612,601_Default/optimizer-FP32StateAdamWeightDecay/Mul-op27,Default/optimizer-FP32StateAdamWeightDecay/Mul-op27,kernel_graph0,{15750},15750,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15746,15771,0x12c1b8c15000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15612,602_Default/learning_rate-CosineWithWarmUpLR/Add-op23,Default/learning_rate-CosineWithWarmUpLR/Add-op23,kernel_graph0,{15759-15769},15769,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15750,15769,0x12c1a3414c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15613,603_Default/learning_rate-CosineWithWarmUpLR/Add-op24,Default/learning_rate-CosineWithWarmUpLR/Add-op24,kernel_graph0,{15754-15767},15767,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15754,15758,0x12c1ae014e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15614,604_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op5,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op5,kernel_graph0,{15756},15756,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15756,15761,0x12c1c3815200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15615,605_Default/learning_rate-CosineWithWarmUpLR/Add-op25,Default/learning_rate-CosineWithWarmUpLR/Add-op25,kernel_graph0,{15759},15759,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15759,15764,0x12c1ae014e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15615,606_Default/optimizer-FP32StateAdamWeightDecay/Div-op5,Default/optimizer-FP32StateAdamWeightDecay/Div-op5,kernel_graph0,{15762},15762,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15762,15767,0x12c1c3815200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15616,607_Default/optimizer-FP32StateAdamWeightDecay/Mul-op29,Default/optimizer-FP32StateAdamWeightDecay/Mul-op29,kernel_graph0,{15765},15765,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15765,15773,0x12c1ae014e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15617,608_Default/optimizer-FP32StateAdamWeightDecay/Sub-op5,Default/optimizer-FP32StateAdamWeightDecay/Sub-op5,kernel_graph0,{15771},15771,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15774,15780,0x12c1a3414c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15622,612_Default/optimizer-FP32StateAdamWeightDecay/Mul-op30,Default/optimizer-FP32StateAdamWeightDecay/Mul-op30,kernel_graph0,{15778},15778,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15776,15784,0x12c1ae014e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15622,613_Default/optimizer-FP32StateAdamWeightDecay/Mul-op32,Default/optimizer-FP32StateAdamWeightDecay/Mul-op32,kernel_graph0,{15782},15782,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15778,15803,0x12c1b8c15000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15623,614_Default/learning_rate-CosineWithWarmUpLR/Add-op26,Default/learning_rate-CosineWithWarmUpLR/Add-op26,kernel_graph0,{15791-15801},15801,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15782,15801,0x12c1a3414c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15625,615_Default/learning_rate-CosineWithWarmUpLR/Add-op27,Default/learning_rate-CosineWithWarmUpLR/Add-op27,kernel_graph0,{15786-15799},15799,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15786,15790,0x12c1ae014e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15626,616_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op6,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op6,kernel_graph0,{15788},15788,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15788,15793,0x12c1c3815200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15627,617_Default/learning_rate-CosineWithWarmUpLR/Add-op28,Default/learning_rate-CosineWithWarmUpLR/Add-op28,kernel_graph0,{15791},15791,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15791,15796,0x12c1ae014e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15628,618_Default/optimizer-FP32StateAdamWeightDecay/Div-op6,Default/optimizer-FP32StateAdamWeightDecay/Div-op6,kernel_graph0,{15794},15794,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15794,15799,0x12c1c3815200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15628,619_Default/optimizer-FP32StateAdamWeightDecay/Mul-op34,Default/optimizer-FP32StateAdamWeightDecay/Mul-op34,kernel_graph0,{15797},15797,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15797,15805,0x12c1ae014e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15629,620_Default/optimizer-FP32StateAdamWeightDecay/Sub-op6,Default/optimizer-FP32StateAdamWeightDecay/Sub-op6,kernel_graph0,{15803},15803,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15806,15812,0x12c1a3414c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15633,624_Default/optimizer-FP32StateAdamWeightDecay/Mul-op35,Default/optimizer-FP32StateAdamWeightDecay/Mul-op35,kernel_graph0,{15810},15810,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15808,15816,0x12c1ae014e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15634,625_Default/optimizer-FP32StateAdamWeightDecay/Mul-op37,Default/optimizer-FP32StateAdamWeightDecay/Mul-op37,kernel_graph0,{15814},15814,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15810,15835,0x12c1b8c15000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15635,626_Default/learning_rate-CosineWithWarmUpLR/Add-op29,Default/learning_rate-CosineWithWarmUpLR/Add-op29,kernel_graph0,{15823-15833},15833,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15814,15833,0x12c1a3414c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15636,627_Default/learning_rate-CosineWithWarmUpLR/Add-op30,Default/learning_rate-CosineWithWarmUpLR/Add-op30,kernel_graph0,{15818-15831},15831,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15818,15822,0x12c1ae014e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15637,628_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op7,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op7,kernel_graph0,{15820},15820,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15820,15825,0x12c208812c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15638,629_Default/learning_rate-CosineWithWarmUpLR/Add-op31,Default/learning_rate-CosineWithWarmUpLR/Add-op31,kernel_graph0,{15823},15823,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15823,15828,0x12c1ae014e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15639,630_Default/optimizer-FP32StateAdamWeightDecay/Div-op7,Default/optimizer-FP32StateAdamWeightDecay/Div-op7,kernel_graph0,{15826},15826,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15826,15831,0x12c208812c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15639,631_Default/optimizer-FP32StateAdamWeightDecay/Mul-op39,Default/optimizer-FP32StateAdamWeightDecay/Mul-op39,kernel_graph0,{15829},15829,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15829,15837,0x12c1ae014e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15640,632_Default/optimizer-FP32StateAdamWeightDecay/Sub-op7,Default/optimizer-FP32StateAdamWeightDecay/Sub-op7,kernel_graph0,{15835},15835,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15838,15844,0x12c1e7936800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15642,636_Default/optimizer-FP32StateAdamWeightDecay/Mul-op40,Default/optimizer-FP32StateAdamWeightDecay/Mul-op40,kernel_graph0,{15842},15842,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15840,15848,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15642,637_Default/optimizer-FP32StateAdamWeightDecay/Mul-op42,Default/optimizer-FP32StateAdamWeightDecay/Mul-op42,kernel_graph0,{15846},15846,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15842,15867,0x12c1f5c12600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15642,638_Default/learning_rate-CosineWithWarmUpLR/Add-op32,Default/learning_rate-CosineWithWarmUpLR/Add-op32,kernel_graph0,{15855-15865},15865,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15846,15865,0x12c1e7936800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15643,639_Default/learning_rate-CosineWithWarmUpLR/Add-op33,Default/learning_rate-CosineWithWarmUpLR/Add-op33,kernel_graph0,{15850-15863},15863,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15850,15854,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15644,640_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op8,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op8,kernel_graph0,{15852},15852,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15852,15857,0x12c23a20fa00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15644,641_Default/learning_rate-CosineWithWarmUpLR/Add-op34,Default/learning_rate-CosineWithWarmUpLR/Add-op34,kernel_graph0,{15855},15855,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15855,15860,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15644,642_Default/optimizer-FP32StateAdamWeightDecay/Div-op8,Default/optimizer-FP32StateAdamWeightDecay/Div-op8,kernel_graph0,{15858},15858,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15858,15863,0x12c23a20fa00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15646,643_Default/optimizer-FP32StateAdamWeightDecay/Mul-op44,Default/optimizer-FP32StateAdamWeightDecay/Mul-op44,kernel_graph0,{15861},15861,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15861,15869,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15647,644_Default/optimizer-FP32StateAdamWeightDecay/Sub-op8,Default/optimizer-FP32StateAdamWeightDecay/Sub-op8,kernel_graph0,{15867},15867,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15870,15876,0x12c1e7936800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15649,648_Default/optimizer-FP32StateAdamWeightDecay/Mul-op45,Default/optimizer-FP32StateAdamWeightDecay/Mul-op45,kernel_graph0,{15874},15874,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15872,15880,0x12c23a20fa00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15650,649_Default/optimizer-FP32StateAdamWeightDecay/Mul-op47,Default/optimizer-FP32StateAdamWeightDecay/Mul-op47,kernel_graph0,{15878},15878,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15874,15899,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15651,650_Default/learning_rate-CosineWithWarmUpLR/Add-op35,Default/learning_rate-CosineWithWarmUpLR/Add-op35,kernel_graph0,{15887-15897},15897,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15878,15897,0x12c1f5c12600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15651,651_Default/learning_rate-CosineWithWarmUpLR/Add-op36,Default/learning_rate-CosineWithWarmUpLR/Add-op36,kernel_graph0,{15882-15895},15895,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15882,15886,0x12c1e7936800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15654,652_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op9,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op9,kernel_graph0,{15884},15884,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15884,15889,0x12c23a20fa00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15655,653_Default/learning_rate-CosineWithWarmUpLR/Add-op37,Default/learning_rate-CosineWithWarmUpLR/Add-op37,kernel_graph0,{15887},15887,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15887,15892,0x12c1e7936800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15655,654_Default/optimizer-FP32StateAdamWeightDecay/Div-op9,Default/optimizer-FP32StateAdamWeightDecay/Div-op9,kernel_graph0,{15890},15890,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15890,15895,0x12c23a20fa00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15658,655_Default/optimizer-FP32StateAdamWeightDecay/Mul-op49,Default/optimizer-FP32StateAdamWeightDecay/Mul-op49,kernel_graph0,{15893},15893,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15893,15901,0x12c1e7936800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15659,656_Default/optimizer-FP32StateAdamWeightDecay/Sub-op9,Default/optimizer-FP32StateAdamWeightDecay/Sub-op9,kernel_graph0,{15899},15899,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15902,15908,0x12c1e7936800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15661,660_Default/optimizer-FP32StateAdamWeightDecay/Mul-op50,Default/optimizer-FP32StateAdamWeightDecay/Mul-op50,kernel_graph0,{15906},15906,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15904,15912,0x12c23a20fa00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15663,661_Default/optimizer-FP32StateAdamWeightDecay/Mul-op52,Default/optimizer-FP32StateAdamWeightDecay/Mul-op52,kernel_graph0,{15910},15910,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15906,15931,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15664,662_Default/learning_rate-CosineWithWarmUpLR/Add-op38,Default/learning_rate-CosineWithWarmUpLR/Add-op38,kernel_graph0,{15919-15929},15929,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15910,15929,0x12c1f5c12600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15664,663_Default/learning_rate-CosineWithWarmUpLR/Add-op39,Default/learning_rate-CosineWithWarmUpLR/Add-op39,kernel_graph0,{15914-15927},15927,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15914,15918,0x12c23a20fa00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15666,664_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op10,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op10,kernel_graph0,{15916},15916,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15916,15921,0x12c1e3936600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15667,665_Default/learning_rate-CosineWithWarmUpLR/Add-op40,Default/learning_rate-CosineWithWarmUpLR/Add-op40,kernel_graph0,{15919},15919,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15919,15924,0x12c23a20fa00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15667,666_Default/optimizer-FP32StateAdamWeightDecay/Div-op10,Default/optimizer-FP32StateAdamWeightDecay/Div-op10,kernel_graph0,{15922},15922,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15922,15927,0x12c1e3936600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15669,667_Default/optimizer-FP32StateAdamWeightDecay/Mul-op54,Default/optimizer-FP32StateAdamWeightDecay/Mul-op54,kernel_graph0,{15925},15925,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15925,15933,0x12c23a20fa00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15670,668_Default/optimizer-FP32StateAdamWeightDecay/Sub-op10,Default/optimizer-FP32StateAdamWeightDecay/Sub-op10,kernel_graph0,{15931},15931,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15934,15940,0x12c23a20fa00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15672,672_Default/optimizer-FP32StateAdamWeightDecay/Mul-op55,Default/optimizer-FP32StateAdamWeightDecay/Mul-op55,kernel_graph0,{15938},15938,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15936,15944,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15672,673_Default/optimizer-FP32StateAdamWeightDecay/Mul-op57,Default/optimizer-FP32StateAdamWeightDecay/Mul-op57,kernel_graph0,{15942},15942,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15938,15963,0x12c1f5c12600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15673,674_Default/learning_rate-CosineWithWarmUpLR/Add-op41,Default/learning_rate-CosineWithWarmUpLR/Add-op41,kernel_graph0,{15951-15961},15961,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15942,15961,0x12c23a20fa00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15673,675_Default/learning_rate-CosineWithWarmUpLR/Add-op42,Default/learning_rate-CosineWithWarmUpLR/Add-op42,kernel_graph0,{15946-15959},15959,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15946,15950,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15674,676_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op11,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op11,kernel_graph0,{15948},15948,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15948,15953,0x12c204812a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15674,677_Default/learning_rate-CosineWithWarmUpLR/Add-op43,Default/learning_rate-CosineWithWarmUpLR/Add-op43,kernel_graph0,{15951},15951,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15951,15956,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15674,678_Default/optimizer-FP32StateAdamWeightDecay/Div-op11,Default/optimizer-FP32StateAdamWeightDecay/Div-op11,kernel_graph0,{15954},15954,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15954,15959,0x12c204812a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15675,679_Default/optimizer-FP32StateAdamWeightDecay/Mul-op59,Default/optimizer-FP32StateAdamWeightDecay/Mul-op59,kernel_graph0,{15957},15957,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15957,15965,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15676,680_Default/optimizer-FP32StateAdamWeightDecay/Sub-op11,Default/optimizer-FP32StateAdamWeightDecay/Sub-op11,kernel_graph0,{15963},15963,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15966,15972,0x12c204812a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15678,684_Default/optimizer-FP32StateAdamWeightDecay/Mul-op60,Default/optimizer-FP32StateAdamWeightDecay/Mul-op60,kernel_graph0,{15970},15970,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15968,15976,0x12c19f414a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15679,685_Default/optimizer-FP32StateAdamWeightDecay/Mul-op62,Default/optimizer-FP32StateAdamWeightDecay/Mul-op62,kernel_graph0,{15974},15974,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15970,15995,0x12c1aa014c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15679,686_Default/learning_rate-CosineWithWarmUpLR/Add-op44,Default/learning_rate-CosineWithWarmUpLR/Add-op44,kernel_graph0,{15983-15993},15993,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15974,15993,0x12c204812a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15680,687_Default/learning_rate-CosineWithWarmUpLR/Add-op45,Default/learning_rate-CosineWithWarmUpLR/Add-op45,kernel_graph0,{15978-15991},15991,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15978,15982,0x12c19f414a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15681,688_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op12,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op12,kernel_graph0,{15980},15980,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15980,15985,0x12c22361f800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15681,689_Default/learning_rate-CosineWithWarmUpLR/Add-op46,Default/learning_rate-CosineWithWarmUpLR/Add-op46,kernel_graph0,{15983},15983,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15983,15988,0x12c19f414a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15682,690_Default/optimizer-FP32StateAdamWeightDecay/Div-op12,Default/optimizer-FP32StateAdamWeightDecay/Div-op12,kernel_graph0,{15986},15986,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15986,15991,0x12c22361f800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15683,691_Default/optimizer-FP32StateAdamWeightDecay/Mul-op64,Default/optimizer-FP32StateAdamWeightDecay/Mul-op64,kernel_graph0,{15989},15989,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15989,15997,0x12c19f414a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15685,692_Default/optimizer-FP32StateAdamWeightDecay/Sub-op12,Default/optimizer-FP32StateAdamWeightDecay/Sub-op12,kernel_graph0,{15995},15995,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 15998,16004,0x12c22361f800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15690,696_Default/optimizer-FP32StateAdamWeightDecay/Mul-op65,Default/optimizer-FP32StateAdamWeightDecay/Mul-op65,kernel_graph0,{16002},16002,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16000,16008,0x12c204812a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15691,697_Default/optimizer-FP32StateAdamWeightDecay/Mul-op67,Default/optimizer-FP32StateAdamWeightDecay/Mul-op67,kernel_graph0,{16006},16006,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16002,16027,0x12c19f414a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15691,698_Default/learning_rate-CosineWithWarmUpLR/Add-op47,Default/learning_rate-CosineWithWarmUpLR/Add-op47,kernel_graph0,{16015-16025},16025,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16006,16025,0x12c22361f800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15692,699_Default/learning_rate-CosineWithWarmUpLR/Add-op48,Default/learning_rate-CosineWithWarmUpLR/Add-op48,kernel_graph0,{16010-16023},16023,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16010,16014,0x12c22e223c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15693,700_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op13,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op13,kernel_graph0,{16012},16012,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16012,16017,0x12c204812a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15695,701_Default/learning_rate-CosineWithWarmUpLR/Add-op49,Default/learning_rate-CosineWithWarmUpLR/Add-op49,kernel_graph0,{16015},16015,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16015,16020,0x12c22e223c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15695,702_Default/optimizer-FP32StateAdamWeightDecay/Div-op13,Default/optimizer-FP32StateAdamWeightDecay/Div-op13,kernel_graph0,{16018},16018,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16018,16023,0x12c204812a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15696,703_Default/optimizer-FP32StateAdamWeightDecay/Mul-op69,Default/optimizer-FP32StateAdamWeightDecay/Mul-op69,kernel_graph0,{16021},16021,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16021,16029,0x12c22e223c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15697,704_Default/optimizer-FP32StateAdamWeightDecay/Sub-op13,Default/optimizer-FP32StateAdamWeightDecay/Sub-op13,kernel_graph0,{16027},16027,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16030,16036,0x12c22361f800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15701,708_Default/optimizer-FP32StateAdamWeightDecay/Mul-op70,Default/optimizer-FP32StateAdamWeightDecay/Mul-op70,kernel_graph0,{16034},16034,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16032,16040,0x12c22e223c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15702,709_Default/optimizer-FP32StateAdamWeightDecay/Mul-op72,Default/optimizer-FP32StateAdamWeightDecay/Mul-op72,kernel_graph0,{16038},16038,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16034,16059,0x12c204812a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15703,710_Default/learning_rate-CosineWithWarmUpLR/Add-op50,Default/learning_rate-CosineWithWarmUpLR/Add-op50,kernel_graph0,{16047-16057},16057,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16038,16057,0x12c22361f800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15703,711_Default/learning_rate-CosineWithWarmUpLR/Add-op51,Default/learning_rate-CosineWithWarmUpLR/Add-op51,kernel_graph0,{16042-16055},16055,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16042,16046,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15704,712_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op14,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op14,kernel_graph0,{16044},16044,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16044,16049,0x12c22e223c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15704,713_Default/learning_rate-CosineWithWarmUpLR/Add-op52,Default/learning_rate-CosineWithWarmUpLR/Add-op52,kernel_graph0,{16047},16047,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16047,16052,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15704,714_Default/optimizer-FP32StateAdamWeightDecay/Div-op14,Default/optimizer-FP32StateAdamWeightDecay/Div-op14,kernel_graph0,{16050},16050,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16050,16055,0x12c22e223c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15705,715_Default/optimizer-FP32StateAdamWeightDecay/Mul-op74,Default/optimizer-FP32StateAdamWeightDecay/Mul-op74,kernel_graph0,{16053},16053,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16053,16061,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15706,716_Default/optimizer-FP32StateAdamWeightDecay/Sub-op14,Default/optimizer-FP32StateAdamWeightDecay/Sub-op14,kernel_graph0,{16059},16059,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16062,16068,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15708,720_Default/optimizer-FP32StateAdamWeightDecay/Mul-op75,Default/optimizer-FP32StateAdamWeightDecay/Mul-op75,kernel_graph0,{16066},16066,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16064,16072,0x12c19f414a00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15708,721_Default/optimizer-FP32StateAdamWeightDecay/Mul-op77,Default/optimizer-FP32StateAdamWeightDecay/Mul-op77,kernel_graph0,{16070},16070,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16066,16091,0x12c1be814c00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15708,722_Default/learning_rate-CosineWithWarmUpLR/Add-op53,Default/learning_rate-CosineWithWarmUpLR/Add-op53,kernel_graph0,{16079-16089},16089,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16070,16089,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15710,723_Default/learning_rate-CosineWithWarmUpLR/Add-op54,Default/learning_rate-CosineWithWarmUpLR/Add-op54,kernel_graph0,{16074-16087},16087,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16074,16078,0x12c180014800,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15711,724_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op15,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op15,kernel_graph0,{16076},16076,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16076,16081,0x12c19f414a00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15711,725_Default/learning_rate-CosineWithWarmUpLR/Add-op55,Default/learning_rate-CosineWithWarmUpLR/Add-op55,kernel_graph0,{16079},16079,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16079,16084,0x12c180014800,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15712,726_Default/optimizer-FP32StateAdamWeightDecay/Div-op15,Default/optimizer-FP32StateAdamWeightDecay/Div-op15,kernel_graph0,{16082},16082,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16082,16087,0x12c19f414a00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15713,727_Default/optimizer-FP32StateAdamWeightDecay/Mul-op79,Default/optimizer-FP32StateAdamWeightDecay/Mul-op79,kernel_graph0,{16085},16085,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16085,16093,0x12c180014800,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15713,728_Default/optimizer-FP32StateAdamWeightDecay/Sub-op15,Default/optimizer-FP32StateAdamWeightDecay/Sub-op15,kernel_graph0,{16091},16091,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16094,16100,0x12c22361f800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15718,732_Default/optimizer-FP32StateAdamWeightDecay/Mul-op80,Default/optimizer-FP32StateAdamWeightDecay/Mul-op80,kernel_graph0,{16098},16098,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16096,16104,0x12c223623a00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15719,733_Default/optimizer-FP32StateAdamWeightDecay/Mul-op82,Default/optimizer-FP32StateAdamWeightDecay/Mul-op82,kernel_graph0,{16102},16102,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16098,16123,0x12c223627c00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15719,734_Default/learning_rate-CosineWithWarmUpLR/Add-op56,Default/learning_rate-CosineWithWarmUpLR/Add-op56,kernel_graph0,{16111-16121},16121,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16102,16121,0x12c22361b600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15722,735_Default/learning_rate-CosineWithWarmUpLR/Add-op57,Default/learning_rate-CosineWithWarmUpLR/Add-op57,kernel_graph0,{16106-16119},16119,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16106,16110,0x12c22361f800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15723,736_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op16,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op16,kernel_graph0,{16108},16108,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16108,16113,0x12c223623a00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15723,737_Default/learning_rate-CosineWithWarmUpLR/Add-op58,Default/learning_rate-CosineWithWarmUpLR/Add-op58,kernel_graph0,{16111},16111,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16111,16116,0x12c22361f800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15724,738_Default/optimizer-FP32StateAdamWeightDecay/Div-op16,Default/optimizer-FP32StateAdamWeightDecay/Div-op16,kernel_graph0,{16114},16114,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16114,16119,0x12c223623a00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15725,739_Default/optimizer-FP32StateAdamWeightDecay/Mul-op84,Default/optimizer-FP32StateAdamWeightDecay/Mul-op84,kernel_graph0,{16117},16117,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16117,16125,0x12c22361f800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15725,740_Default/optimizer-FP32StateAdamWeightDecay/Sub-op16,Default/optimizer-FP32StateAdamWeightDecay/Sub-op16,kernel_graph0,{16123},16123,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16126,16132,0x12c22361b600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15730,744_Default/optimizer-FP32StateAdamWeightDecay/Mul-op85,Default/optimizer-FP32StateAdamWeightDecay/Mul-op85,kernel_graph0,{16130},16130,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16128,16136,0x12c22361f800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15731,745_Default/optimizer-FP32StateAdamWeightDecay/Mul-op87,Default/optimizer-FP32StateAdamWeightDecay/Mul-op87,kernel_graph0,{16134},16134,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16130,16155,0x12c223623a00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15731,746_Default/learning_rate-CosineWithWarmUpLR/Add-op59,Default/learning_rate-CosineWithWarmUpLR/Add-op59,kernel_graph0,{16143-16153},16153,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16134,16153,0x12c223617400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15733,747_Default/learning_rate-CosineWithWarmUpLR/Add-op60,Default/learning_rate-CosineWithWarmUpLR/Add-op60,kernel_graph0,{16138-16151},16151,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16138,16142,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15734,748_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op17,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op17,kernel_graph0,{16140},16140,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16140,16145,0x12c22361b600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15734,749_Default/learning_rate-CosineWithWarmUpLR/Add-op61,Default/learning_rate-CosineWithWarmUpLR/Add-op61,kernel_graph0,{16143},16143,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16143,16148,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15735,750_Default/optimizer-FP32StateAdamWeightDecay/Div-op17,Default/optimizer-FP32StateAdamWeightDecay/Div-op17,kernel_graph0,{16146},16146,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16146,16151,0x12c22361b600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15736,751_Default/optimizer-FP32StateAdamWeightDecay/Mul-op89,Default/optimizer-FP32StateAdamWeightDecay/Mul-op89,kernel_graph0,{16149},16149,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16149,16157,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15736,752_Default/optimizer-FP32StateAdamWeightDecay/Sub-op17,Default/optimizer-FP32StateAdamWeightDecay/Sub-op17,kernel_graph0,{16155},16155,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16158,16164,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15738,756_Default/optimizer-FP32StateAdamWeightDecay/Mul-op90,Default/optimizer-FP32StateAdamWeightDecay/Mul-op90,kernel_graph0,{16162},16162,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16160,16168,0x12c223617400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15738,757_Default/optimizer-FP32StateAdamWeightDecay/Mul-op92,Default/optimizer-FP32StateAdamWeightDecay/Mul-op92,kernel_graph0,{16166},16166,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16162,16187,0x12c22361b600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15739,758_Default/learning_rate-CosineWithWarmUpLR/Add-op62,Default/learning_rate-CosineWithWarmUpLR/Add-op62,kernel_graph0,{16175-16185},16185,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16166,16185,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15740,759_Default/learning_rate-CosineWithWarmUpLR/Add-op63,Default/learning_rate-CosineWithWarmUpLR/Add-op63,kernel_graph0,{16170-16183},16183,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16170,16174,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15740,760_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op18,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op18,kernel_graph0,{16172},16172,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16172,16177,0x12c22361f800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15740,761_Default/learning_rate-CosineWithWarmUpLR/Add-op64,Default/learning_rate-CosineWithWarmUpLR/Add-op64,kernel_graph0,{16175},16175,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16175,16180,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15741,762_Default/optimizer-FP32StateAdamWeightDecay/Div-op18,Default/optimizer-FP32StateAdamWeightDecay/Div-op18,kernel_graph0,{16178},16178,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16178,16183,0x12c22361f800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15742,763_Default/optimizer-FP32StateAdamWeightDecay/Mul-op94,Default/optimizer-FP32StateAdamWeightDecay/Mul-op94,kernel_graph0,{16181},16181,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16181,16189,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15743,764_Default/optimizer-FP32StateAdamWeightDecay/Sub-op18,Default/optimizer-FP32StateAdamWeightDecay/Sub-op18,kernel_graph0,{16187},16187,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16190,16196,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15745,768_Default/optimizer-FP32StateAdamWeightDecay/Mul-op97,Default/optimizer-FP32StateAdamWeightDecay/Mul-op97,kernel_graph0,{16194},16194,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16192,16200,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15746,769_Default/optimizer-FP32StateAdamWeightDecay/Mul-op95,Default/optimizer-FP32StateAdamWeightDecay/Mul-op95,kernel_graph0,{16198},16198,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16194,16217,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15747,770_Default/learning_rate-CosineWithWarmUpLR/Add-op66,Default/learning_rate-CosineWithWarmUpLR/Add-op66,kernel_graph0,{16202-16215},16215,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16198,16219,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15749,771_Default/learning_rate-CosineWithWarmUpLR/Add-op65,Default/learning_rate-CosineWithWarmUpLR/Add-op65,kernel_graph0,{16207-16217},16217,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16202,16206,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15750,772_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op19,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op19,kernel_graph0,{16204},16204,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16204,16209,0x12c1f1c16600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15751,773_Default/learning_rate-CosineWithWarmUpLR/Add-op67,Default/learning_rate-CosineWithWarmUpLR/Add-op67,kernel_graph0,{16207},16207,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16207,16212,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15751,774_Default/optimizer-FP32StateAdamWeightDecay/Div-op19,Default/optimizer-FP32StateAdamWeightDecay/Div-op19,kernel_graph0,{16210},16210,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16210,16215,0x12c1f1c16600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15754,775_Default/optimizer-FP32StateAdamWeightDecay/Mul-op99,Default/optimizer-FP32StateAdamWeightDecay/Mul-op99,kernel_graph0,{16213},16213,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16213,16221,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15755,776_Default/optimizer-FP32StateAdamWeightDecay/Sub-op19,Default/optimizer-FP32StateAdamWeightDecay/Sub-op19,kernel_graph0,{16219},16219,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16222,16228,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15757,780_Default/optimizer-FP32StateAdamWeightDecay/Mul-op102,Default/optimizer-FP32StateAdamWeightDecay/Mul-op102,kernel_graph0,{16226},16226,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16224,16232,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15759,781_Default/optimizer-FP32StateAdamWeightDecay/Mul-op100,Default/optimizer-FP32StateAdamWeightDecay/Mul-op100,kernel_graph0,{16230},16230,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16226,16254,0x12c180010600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15760,782_Default/learning_rate-CosineWithWarmUpLR/Add-op69,Default/learning_rate-CosineWithWarmUpLR/Add-op69,kernel_graph0,{16234-16252},16252,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16230,16252,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15760,783_Default/learning_rate-CosineWithWarmUpLR/Add-op68,Default/learning_rate-CosineWithWarmUpLR/Add-op68,kernel_graph0,{16239-16250},16250,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16234,16238,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15762,784_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op20,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op20,kernel_graph0,{16236},16236,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16236,16241,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15763,785_Default/learning_rate-CosineWithWarmUpLR/Add-op70,Default/learning_rate-CosineWithWarmUpLR/Add-op70,kernel_graph0,{16239},16239,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16239,16245,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15763,786_Default/optimizer-FP32StateAdamWeightDecay/Div-op20,Default/optimizer-FP32StateAdamWeightDecay/Div-op20,kernel_graph0,{16242},16242,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16242,16248,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15765,787_Default/optimizer-FP32StateAdamWeightDecay/Mul-op104,Default/optimizer-FP32StateAdamWeightDecay/Mul-op104,kernel_graph0,{16246},16246,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16246,16250,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15766,788_Default/optimizer-FP32StateAdamWeightDecay/Sub-op20,Default/optimizer-FP32StateAdamWeightDecay/Sub-op20,kernel_graph0,{16248},16248,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16255,18554,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,15768,792_Default/TensorMove-op2,Default/TensorMove-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16259,18552,0x12c75d35ec00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/output_actor.cc,689,Other,16257,kernel_graph_0_OutputActor,UpdateOutputDeviceAddress,,{},,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 16265,16669,0x12c75d35ee00,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Kernel,16264,0_Default/network-LlamaForCausalLM/StridedSlice-op0,inputs0,0_Default/network-LlamaForCausalLM/StridedSlice-op0,{16266-16667},16667,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,1,0, 16266,17248,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16264,0_Default/network-LlamaForCausalLM/StridedSlice-op0,inputs0,0_Default/network-LlamaForCausalLM/StridedSlice-op0,{16268-16270-17245},17245,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16268,16274,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16266,1_Default/network-LlamaForCausalLM/model-LlamaModel/tok_embeddings-LlamaEmbedding/Gather-op0,Default/network-LlamaForCausalLM/model-LlamaModel/tok_embeddings-LlamaEmbedding/Gather-op0,kernel_graph0,{16272},16272,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16270,16280,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,8704,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16267,2_Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/NotEqual-op0,Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/NotEqual-op0,kernel_graph0,{16275-16278},16278,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16272,17226,0x12c18800c600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16267,3_Default/network-LlamaForCausalLM/model-LlamaModel/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/Cast-op0,kernel_graph0,{16284-16413-17224},17224,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16275,16703,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16267,4_Default/network-LlamaForCausalLM/Cast-op1,Default/network-LlamaForCausalLM/Cast-op1,kernel_graph0,{16276-16295-16656-16700},16700,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16278,16309,0x12c180014600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16268,6_Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Cast-op0,kernel_graph0,{16303-16307},16307,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16281,17152,0x12c180018800,0,DefaultEnhancedAscendMemoryPool,524800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16268,7_Default/network-LlamaForCausalLM/model-LlamaModel/freqs_mgr-FreqsMgr/StridedSlice-op0,Default/network-LlamaForCausalLM/model-LlamaModel/freqs_mgr-FreqsMgr/StridedSlice-op0,kernel_graph0,{16282-16346-16369-16499-16514-16887-16908-17129-17150},17150,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16284,16290,0x12c18c00c800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16269,9_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op0,kernel_graph0,{16288},16288,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16286,16294,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16269,10_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{16292},16292,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16287,17233,0x12c180098a00,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16269,10_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{17229},17229,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16288,16291,0x12c1800a0c00,0,DefaultEnhancedAscendMemoryPool,16777728,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16269,10_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16292,17204,0x12c1800a0c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16270,11_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op1,kernel_graph0,{16317-16322-16337-16360-17174-17188-17202},17202,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16295,16303,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16271,12_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/ReduceSum-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/ReduceSum-op0,kernel_graph0,{16301},16301,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16297,16299,0x12c180008800,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,16297,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16298,16300,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,16298,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16301,16714,0x12c180008800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16271,13_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Add-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Add-op0,kernel_graph0,{16644-16711},16711,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16305,16310,0x12c1840a0e00,0,DefaultEnhancedAscendMemoryPool,2097664,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16273,15_Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/ExpandDims-op0,Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/ExpandDims-op0,kernel_graph0,{16307},16307,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16307,16313,0x12c1842a1000,0,DefaultEnhancedAscendMemoryPool,16777728,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16273,16_Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Mul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Mul-op0,kernel_graph0,{16311},16311,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16311,16317,0x12c1852a1200,0,DefaultEnhancedAscendMemoryPool,16777728,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16275,17_Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Sub-op0,Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Sub-op0,kernel_graph0,{16313-16315},16315,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16315,17110,0x12c1840a0e00,0,DefaultEnhancedAscendMemoryPool,8389120,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16276,19_Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Cast-op1,kernel_graph0,{16398-16545-16863-17104},17104,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16319,17198,0x12c1848a1000,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16277,21_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/Cast-op0,kernel_graph0,{16322-17195},17195,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16321,16329,0x12c18c00c800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16278,22_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,kernel_graph0,{16324-16327},16327,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16322,16324,0x12c1868a1200,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16278,22_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16326,16333,0x12c19000ca00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16279,24_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,kernel_graph0,{16331},16331,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16327,16330,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16279,24_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16331,16350,0x12c19400cc00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16281,25_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op2,kernel_graph0,{16346-16348},16348,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16334,17213,0x12c18c00c800,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16282,26_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/Cast-op0,kernel_graph0,{16337-17210},17210,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16336,16344,0x12c18e00ca00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16282,27_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,kernel_graph0,{16339-16342},16342,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16337,16339,0x12c1868a1200,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16282,27_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16341,16353,0x12c19c00ce00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16282,29_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,kernel_graph0,{16351},16351,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16342,16345,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16282,29_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16346,16389,0x12c1a000d000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16282,30_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op2,kernel_graph0,{16387},16387,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16348,16386,0x12c1a800d200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16283,31_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op1,kernel_graph0,{16384},16384,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16351,16373,0x12c18e00ca00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16284,32_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op0,kernel_graph0,{16369-16371},16371,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16354,17155,0x12c1868a1200,0,DefaultEnhancedAscendMemoryPool,524800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16285,33_Default/network-LlamaForCausalLM/model-LlamaModel/freqs_mgr-FreqsMgr/StridedSlice-op1,Default/network-LlamaForCausalLM/model-LlamaModel/freqs_mgr-FreqsMgr/StridedSlice-op1,kernel_graph0,{16355-16374-16384-16504-16519-16889-16910-17131-17153},17153,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16357,17184,0x12c19600cc00,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16288,35_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/Cast-op0,kernel_graph0,{16360-17181},17181,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16359,16367,0x12c19800ce00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16289,36_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,kernel_graph0,{16362-16365},16365,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16360,16362,0x12c186921400,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16289,36_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16364,17108,0x12c1b000d400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16289,38_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,kernel_graph0,{16398-17104},17104,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16365,16368,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16289,38_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16369,16379,0x12c19800ce00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16292,39_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op0,kernel_graph0,{16377},16377,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16371,16376,0x12c1b400d600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16293,40_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op0,kernel_graph0,{16374},16374,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16374,16380,0x12c18e00ca00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16293,41_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op1,kernel_graph0,{16377},16377,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16377,16383,0x12c1b400d600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16295,42_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op0,kernel_graph0,{16381},16381,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16381,17106,0x12c18e00ca00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16296,43_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op1,kernel_graph0,{16398-17104},17104,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16384,16390,0x12c19800ce00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16296,44_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op3,kernel_graph0,{16387},16387,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16387,16393,0x12c1a800d200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16296,45_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op1,kernel_graph0,{16391},16391,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16391,17107,0x12c19800ce00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16296,46_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op3,kernel_graph0,{16398-17104},17104,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16394,17111,0x12c186921400,0,DefaultEnhancedAscendMemoryPool,8389120,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16297,47_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{17104},17104,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16395,17112,0x12c187121600,0,DefaultEnhancedAscendMemoryPool,8389120,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16297,47_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{17104},17104,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16396,17113,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16297,47_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{17104},17104,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16397,17114,0x12c19c00d000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16297,47_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{16402-17104},17104,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16398,16400,0x12c1a000d200,0,DefaultEnhancedAscendMemoryPool,84936192,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16297,47_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16401,17084,0x12c1a000d200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16298,48_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,kernel_graph0,{16404-16409-17082},17082,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16402,16404,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16298,48_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16406,17092,0x12c19200cc00,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16299,50_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/Cast-op0,kernel_graph0,{16409-17090},17090,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16408,16415,0x12c1a400d400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16301,51_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,kernel_graph0,{16411-16413},16413,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16409,16411,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16301,51_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16413,17064,0x12c1a800d600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16302,53_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/Add-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/Add-op0,kernel_graph0,{16416-16454-17062},17062,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16416,16422,0x12c1b400d600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16303,54_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op0,kernel_graph0,{16420},16420,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16418,16426,0x12c1bc00d800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16304,55_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{16424},16424,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16419,17071,0x12c187921800,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16304,55_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{17067},17067,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16420,16423,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,16777728,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16304,55_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16424,17043,0x12c1a400d400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16305,56_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op1,kernel_graph0,{16426-16431-16438-17027-17041},17041,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16428,17037,0x12c1b400d600,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16307,58_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/Cast-op0,kernel_graph0,{16431-17034},17034,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16430,17019,0x12c1b960d800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16308,59_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,kernel_graph0,{16433-16444-17017},17017,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16431,16433,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16308,59_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16435,17052,0x12c1c420da00,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16311,61_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/Cast-op0,kernel_graph0,{16438-17049},17049,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16437,17025,0x12c1c980dc00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16312,62_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,kernel_graph0,{16441-17022},17022,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16438,16440,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16312,62_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16441,17015,0x12c1d440de00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16312,63_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/SiLU-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/SiLU-op0,kernel_graph0,{16442-16444-17013},17013,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16444,17002,0x12c1df00e000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16313,65_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/Mul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/Mul-op0,kernel_graph0,{16445-16450-17000},17000,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16447,17010,0x12c1e9c0e200,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16315,67_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/Cast-op0,kernel_graph0,{16450-17008},17008,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16449,16456,0x12c1ef20e400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16316,68_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,kernel_graph0,{16452-16454},16454,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16450,16452,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16316,68_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16454,16982,0x12c1f320e600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16318,70_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/Add-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/Add-op1,kernel_graph0,{16457-16560-16980},16980,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16457,16463,0x12c1f720e800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16319,71_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op0,kernel_graph0,{16461},16461,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16459,16467,0x12c1ff20ea00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16320,72_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{16465},16465,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16460,16989,0x12c187929a00,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16320,72_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{16985},16985,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16461,16464,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,16777728,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16320,72_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16465,16960,0x12c1ef20e400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16320,73_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op1,kernel_graph0,{16467-16472-16487-16532-16930-16944-16958},16958,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16469,16954,0x12c1ac00d800,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16323,75_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/Cast-op0,kernel_graph0,{16472-16951},16951,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16471,16479,0x12c1f720e800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16324,76_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,kernel_graph0,{16474-16477},16477,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16472,16474,0x12c1ae00da00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16324,76_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16476,16483,0x12c1fb20ea00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16327,78_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,kernel_graph0,{16481},16481,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16477,16480,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16327,78_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16481,16503,0x12c1ff20ec00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16328,79_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op2,kernel_graph0,{16499-16501},16501,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16484,16969,0x12c1f720e800,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16328,80_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/Cast-op0,kernel_graph0,{16487-16966},16966,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16486,16494,0x12c1f920ea00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16331,81_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,kernel_graph0,{16489-16492},16492,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16487,16489,0x12c1ae00da00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16331,81_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16491,16498,0x12c20720ee00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16332,83_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,kernel_graph0,{16496},16496,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16492,16495,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16332,83_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16496,16518,0x12c20b20f000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16334,84_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op0,kernel_graph0,{16514-16516},16516,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16499,16509,0x12c21320f200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16335,85_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op2,kernel_graph0,{16507},16507,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16501,16506,0x12c21b20f400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16335,86_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op1,kernel_graph0,{16504},16504,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16504,16510,0x12c1f920ea00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16337,87_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op3,kernel_graph0,{16507},16507,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16507,16513,0x12c20120ec00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16338,88_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op1,kernel_graph0,{16511},16511,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16511,16866,0x12c1f920ea00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16338,89_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op3,kernel_graph0,{16545-16863},16863,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16514,16524,0x12c1fd20ec00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16339,90_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op0,kernel_graph0,{16522},16522,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16516,16521,0x12c21320f200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16340,91_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op0,kernel_graph0,{16519},16519,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16519,16525,0x12c20520ee00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16342,92_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op1,kernel_graph0,{16522},16522,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16522,16528,0x12c20d20f000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16342,93_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op0,kernel_graph0,{16526},16526,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16526,16865,0x12c1fd20ec00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16343,94_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op1,kernel_graph0,{16545-16863},16863,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16529,16940,0x12c20120ee00,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16346,95_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/Cast-op0,kernel_graph0,{16532-16937},16937,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16531,16539,0x12c20320f000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16346,96_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,kernel_graph0,{16534-16537},16537,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16532,16534,0x12c1ae00da00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16346,96_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16536,16867,0x12c20720f200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16348,98_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,kernel_graph0,{16545-16863},16863,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16537,16540,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16348,98_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16541,16869,0x12c1ae00da00,0,DefaultEnhancedAscendMemoryPool,8389120,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16348,99_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{16863},16863,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16542,16870,0x12c1ae80dc00,0,DefaultEnhancedAscendMemoryPool,8389120,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16348,99_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{16863},16863,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16543,16871,0x12c180008a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16348,99_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{16863},16863,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16544,16872,0x12c20320f000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16348,99_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{16549-16863},16863,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16545,16547,0x12c20b20f400,0,DefaultEnhancedAscendMemoryPool,84936192,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16348,99_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16548,16843,0x12c20b20f400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16349,100_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,kernel_graph0,{16551-16556-16841},16841,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16549,16551,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16349,100_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16553,16851,0x12c20f20f600,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16351,102_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/Cast-op0,kernel_graph0,{16556-16849},16849,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16555,16562,0x12c21120f800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16352,103_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,kernel_graph0,{16558-16560},16560,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16556,16558,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16352,103_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16560,16818,0x12c21520fa00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16354,105_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/Add-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/Add-op0,kernel_graph0,{16563-16601-16816},16816,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16563,16569,0x12c21920fc00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16355,106_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op0,kernel_graph0,{16567},16567,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16565,16573,0x12c22120fe00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16355,107_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{16571},16571,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16566,16825,0x12c187931c00,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16355,107_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{16821},16821,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16567,16570,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,16777728,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16355,107_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16571,16797,0x12c21120f800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16355,108_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op1,kernel_graph0,{16573-16578-16585-16781-16795},16795,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16575,16791,0x12c21920fc00,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16355,110_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/Cast-op0,kernel_graph0,{16578-16788},16788,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16577,16773,0x12c21e80fe00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16356,111_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,kernel_graph0,{16580-16591-16771},16771,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16578,16580,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16356,111_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16582,16806,0x12c229410000,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16358,113_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/Cast-op0,kernel_graph0,{16585-16803},16803,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16584,16779,0x12c22ea10200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16358,114_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,kernel_graph0,{16588-16776},16776,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16585,16587,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16358,114_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16588,16769,0x12c239610400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16360,115_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/SiLU-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/SiLU-op0,kernel_graph0,{16589-16591-16767},16767,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16591,16756,0x12c244210600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16362,117_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/Mul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/Mul-op0,kernel_graph0,{16592-16597-16754},16754,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16594,16764,0x12c24ee10800,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16363,119_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/Cast-op0,kernel_graph0,{16597-16762},16762,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16596,16603,0x12c254410a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16365,120_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,kernel_graph0,{16599-16601},16601,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16597,16599,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16365,120_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16601,16735,0x12c258410c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16366,122_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/Add-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/Add-op1,kernel_graph0,{16604-16733},16733,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16604,16610,0x12c25c410e00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16369,123_Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Cast-op0,kernel_graph0,{16608},16608,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16606,16614,0x12c264411000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16370,124_Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{16612},16612,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16607,16742,0x12c187939e00,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16370,124_Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{16738},16738,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16608,16611,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,16777728,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16370,124_Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16612,16718,0x12c254410a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16370,125_Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Cast-op1,kernel_graph0,{16614-16619-16716},16716,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16616,16727,0x12c25c410e00,0,DefaultEnhancedAscendMemoryPool,262144512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16372,127_Default/network-LlamaForCausalLM/lm_head-Linear/Cast-op0,Default/network-LlamaForCausalLM/lm_head-Linear/Cast-op0,kernel_graph0,{16619-16724},16724,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16618,16624,0x12c26be11000,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16372,128_Default/network-LlamaForCausalLM/lm_head-Linear/MatMul-op0,Default/network-LlamaForCausalLM/lm_head-Linear/MatMul-op0,kernel_graph0,{16622},16622,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16619,16621,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16372,128_Default/network-LlamaForCausalLM/lm_head-Linear/MatMul-op0,Default/network-LlamaForCausalLM/lm_head-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16622,16631,0x12c28b211200,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16374,129_Default/network-LlamaForCausalLM/Cast-op0,Default/network-LlamaForCausalLM/Cast-op0,kernel_graph0,{16626-16629},16629,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16625,16632,0x12c187942000,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16375,130_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/ReduceMax-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/ReduceMax-op0,kernel_graph0,{16629},16629,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16626,16628,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16375,130_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/ReduceMax-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/ReduceMax-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16629,16665,0x12c2c9a11400,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16375,131_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Sub-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Sub-op0,kernel_graph0,{16635-16663},16663,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16633,16646,0x12c180008c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16377,132_Default/Mul-op0,Default/Mul-op0,kernel_graph0,{16644},16644,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16635,16643,0x12c26be11000,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16378,133_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Exp-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Exp-op0,kernel_graph0,{16637},16637,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16637,16662,0x12c187942000,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16378,134_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/ReduceSum-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/ReduceSum-op0,kernel_graph0,{16660},16660,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16639,16641,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,16639,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16640,16642,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,16640,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16644,16650,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16381,135_Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_RealDiv/RealDiv-op0,Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_RealDiv/RealDiv-op0,kernel_graph0,{16648},16648,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16647,16655,0x12c180008c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16382,136_Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_RealDiv/SumExt-op0,Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_RealDiv/SumExt-op0,kernel_graph0,{16651-16653},16653,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16648,16651,0x12c180009600,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16382,136_Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_RealDiv/SumExt-op0,Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_RealDiv/SumExt-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16653,16658,0x12c18794a200,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16384,138_Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_ReduceSum/Tile-op0,Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_ReduceSum/Tile-op0,kernel_graph0,{16656},16656,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16656,16682,0x12c187952400,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16385,139_Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_Mul/Mul-op0,Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_Mul/Mul-op0,kernel_graph0,{16658-16679},16679,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16660,16666,0x12c18794a200,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16387,141_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Log-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Log-op0,kernel_graph0,{16663},16663,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16663,16688,0x12c26be11000,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16388,142_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Sub-op1,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Sub-op1,kernel_graph0,{16674-16686},16686,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16667,16673,0x12c187942000,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16388,143_Default/network-LlamaForCausalLM/StridedSlice-op1,Default/network-LlamaForCausalLM/StridedSlice-op1,kernel_graph0,{16669-16671},16671,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16671,16689,0x12c2aa611200,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16392,145_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/OneHot-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/OneHot-op0,kernel_graph0,{16676-16686},16686,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16674,16678,0x12c2e8e11400,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16392,146_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Exp-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Exp-op0,kernel_graph0,{16676},16676,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16676,16681,0x12c327611600,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16397,147_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Sub-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Sub-op0,kernel_graph0,{16679},16679,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16679,16685,0x12c2e8e11400,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16398,148_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Mul-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Mul-op0,kernel_graph0,{16683},16683,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16683,16726,0x12c327611600,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16399,149_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op0,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op0,kernel_graph0,{16716-16724},16724,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16686,16692,0x12c2e8e11400,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16399,150_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Mul-op1,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Mul-op1,kernel_graph0,{16690},16690,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16690,16699,0x12c26be11000,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16400,151_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Neg-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Neg-op0,kernel_graph0,{16693},16693,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16693,16702,0x12c187942000,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16402,152_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/ReduceSum-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/ReduceSum-op0,kernel_graph0,{16700},16700,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16695,16697,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,16695,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16696,16698,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,16696,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16700,16710,0x12c18794a200,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16403,153_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Mul-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Mul-op0,kernel_graph0,{16704},16704,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16704,16713,0x12c180008c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16403,154_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/ReduceSum-op1,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/ReduceSum-op1,kernel_graph0,{16711},16711,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16706,16708,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,16706,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16707,16709,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,16707,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16711,18547,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16404,155_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/RealDiv-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/RealDiv-op0,kernel_graph0,{18545},18545,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16715,16722,0x12c346a11800,0,DefaultEnhancedAscendMemoryPool,262144512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16405,156_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op29,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op29,kernel_graph0,{16720},16720,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16716,16719,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16405,156_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op29,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op29,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16720,17301,0x12c26be11000,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16406,157_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op29,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op29,kernel_graph0,{17299},17299,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16723,16732,0x12c254410a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16407,158_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op0,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op0,kernel_graph0,{16728-16730},16730,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16724,16728,0x12c346a11800,0,DefaultEnhancedAscendMemoryPool,155190272,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16407,158_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op0,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16730,16740,0x12c25c410e00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16409,160_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op1,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op1,kernel_graph0,{16738},16738,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16733,16741,0x12c28b211200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16410,161_Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Cast-op2,kernel_graph0,{16738},16738,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16736,16746,0x12c254410a00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16411,162_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op0,kernel_graph0,{16744},16744,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16737,17316,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16411,162_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op0,kernel_graph0,{17314},17314,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16738,16743,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,16778752,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16411,162_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16744,16833,0x12c25c410c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16412,163_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op0,kernel_graph0,{16748-16754-16762-16830},16830,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16747,16752,0x12c180008800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16413,164_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Mul-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Mul-op0,kernel_graph0,{16750},16750,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16750,17516,0x12c180008c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16414,166_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Maximum-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Maximum-op0,kernel_graph0,{17514},17514,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16753,16760,0x12c254410a00,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16416,167_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op27,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op27,kernel_graph0,{16758},16758,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16754,16757,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16416,167_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op27,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op27,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16758,17295,0x12c244210600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16417,168_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op27,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op27,kernel_graph0,{17293},17293,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16761,16774,0x12c260410e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16417,169_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op1,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op1,kernel_graph0,{16765-16767-16771},16771,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16762,16765,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16417,169_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op1,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16767,16790,0x12c24ee10800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16421,171_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op0,kernel_graph0,{16769-16781-16788},16788,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16771,16778,0x12c239610400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16424,173_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op1,kernel_graph0,{16774-16776},16776,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16776,16805,0x12c21e80fe00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16425,175_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/Grad_SiLU/SiLUGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/Grad_SiLU/SiLUGrad-op0,kernel_graph0,{16795-16803},16803,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16780,16786,0x12c260410e00,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16426,176_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op28,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op28,kernel_graph0,{16784},16784,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16781,16783,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16426,176_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op28,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op28,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16784,17298,0x12c22ea10200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16427,177_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op28,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op28,kernel_graph0,{17296},17296,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16787,16811,0x12c239610400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16428,178_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op2,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op2,kernel_graph0,{16792-16809},16809,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16788,16792,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16428,178_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op2,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16794,16801,0x12c21920fc00,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16429,180_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op26,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op26,kernel_graph0,{16799},16799,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16795,16798,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16429,180_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op26,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op26,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16799,17292,0x12c260410e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16431,181_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op26,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op26,kernel_graph0,{17290},17290,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16802,16812,0x12c21120f800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16432,182_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op3,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op3,kernel_graph0,{16807-16809},16809,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16803,16807,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16432,182_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op3,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op3,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16809,16815,0x12c23d610600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16433,184_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/AddN-op0,kernel_graph0,{16813},16813,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16813,16823,0x12c24ee10800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16434,185_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op2,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op2,kernel_graph0,{16821},16821,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16816,16824,0x12c239610400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16435,186_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op2,kernel_graph0,{16821},16821,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16819,16829,0x12c21120f800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16436,187_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op1,kernel_graph0,{16827},16827,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16820,17310,0x12c180010600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16436,187_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op1,kernel_graph0,{17308},17308,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16821,16826,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,16778752,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16436,187_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16827,16832,0x12c239610400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16436,188_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op1,kernel_graph0,{16830},16830,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16830,16997,0x12c23d610600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16438,189_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/AddN-op0,kernel_graph0,{16835-16841-16849-16994},16994,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16834,16839,0x12c180008800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16439,190_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Sub-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Sub-op0,kernel_graph0,{16837},16837,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16837,17539,0x12c180009600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16441,192_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/LessEqual-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/LessEqual-op0,kernel_graph0,{17519-17537},17537,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16840,16847,0x12c241610800,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16442,193_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op25,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op25,kernel_graph0,{16845},16845,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16841,16844,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16442,193_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op25,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op25,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16845,17289,0x12c20b20f400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16442,194_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op25,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op25,kernel_graph0,{17287},17287,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16848,16857,0x12c239610400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16442,195_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op4,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op4,kernel_graph0,{16852-16855},16855,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16849,16852,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16442,195_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op4,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op4,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16854,16868,0x12c24ee10800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16444,197_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op0,kernel_graph0,{16863},16863,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16855,16858,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16444,197_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16859,16880,0x12c239610400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16445,198_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,kernel_graph0,{16878},16878,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16860,16877,0x12c252e10a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16445,198_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,kernel_graph0,{16875},16875,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16861,16884,0x12c256e10c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16445,198_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,kernel_graph0,{16882},16882,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16862,16873,0x12c180008800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16445,198_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16863,16874,0x12c20f20f600,0,DefaultEnhancedAscendMemoryPool,515969024,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16445,198_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16875,16891,0x12c1f920ea00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16445,199_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op5,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op5,kernel_graph0,{16887-16889},16889,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16878,16912,0x12c20320f000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16446,200_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op3,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op3,kernel_graph0,{16908-16910},16910,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16881,16939,0x12c239610400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16447,201_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op3,kernel_graph0,{16885-16930-16937},16937,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16882,16885,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16447,201_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op3,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16887,16897,0x12c24ee10800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16448,203_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op4,kernel_graph0,{16895},16895,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16889,16894,0x12c256e10a00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16450,204_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op5,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op5,kernel_graph0,{16892},16892,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16892,16898,0x12c1f920ea00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16451,205_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op1,kernel_graph0,{16895},16895,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16895,16901,0x12c256e10a00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16452,206_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op1,kernel_graph0,{16899},16899,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16899,16905,0x12c24ee10800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16453,207_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op6,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op6,kernel_graph0,{16903},16903,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16902,16953,0x12c1f920ea00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16454,208_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op2,kernel_graph0,{16906-16944-16951},16951,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16903,16906,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16454,208_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16908,16918,0x12c24ee10800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16455,210_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op2,kernel_graph0,{16916},16916,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16910,16915,0x12c256e10a00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16457,211_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op3,kernel_graph0,{16913},16913,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16913,16919,0x12c20320f000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16458,212_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op0,kernel_graph0,{16916},16916,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16916,16922,0x12c256e10a00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16458,213_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op0,kernel_graph0,{16920},16920,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16920,16926,0x12c1fd20ec00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16461,214_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op4,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op4,kernel_graph0,{16924},16924,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16923,16968,0x12c20320f000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16462,215_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op1,kernel_graph0,{16927-16958-16966},16966,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16924,16927,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16462,215_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16929,16935,0x12c241610800,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16465,217_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op24,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op24,kernel_graph0,{16933},16933,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16930,16932,0x12c1ae00da00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16465,217_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op24,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op24,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16933,17286,0x12c1fd20ec00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16466,218_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op24,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op24,kernel_graph0,{17284},17284,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16936,16976,0x12c20720f200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16466,219_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op7,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op7,kernel_graph0,{16941-16972},16972,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16937,16941,0x12c1ae00da00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16466,219_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op7,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op7,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16943,16949,0x12c20120ee00,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16468,221_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op23,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op23,kernel_graph0,{16947},16947,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16944,16946,0x12c1ae00da00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16468,221_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op23,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op23,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16947,17283,0x12c239610400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16469,222_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op23,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op23,kernel_graph0,{17281},17281,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16950,16975,0x12c24ee10800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16470,223_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op6,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op6,kernel_graph0,{16955-16972},16972,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16951,16955,0x12c1ae00da00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16470,223_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op6,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op6,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16957,16964,0x12c20120ee00,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16472,225_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op22,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op22,kernel_graph0,{16962},16962,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16958,16961,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16472,225_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op22,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op22,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16962,17280,0x12c1ef20e400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16473,226_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op22,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op22,kernel_graph0,{17278},17278,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16965,16974,0x12c1f920ea00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16473,227_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op5,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op5,kernel_graph0,{16970-16972},16972,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16966,16970,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16473,227_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op5,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op5,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16972,16979,0x12c20120ee00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16475,229_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/AddN-op0,kernel_graph0,{16977},16977,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16977,16987,0x12c24ee10800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16477,230_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op7,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op7,kernel_graph0,{16985},16985,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16980,16988,0x12c256e10a00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16477,231_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op2,kernel_graph0,{16985},16985,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16983,16993,0x12c1f320e600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16478,232_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op2,kernel_graph0,{16991},16991,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16984,17313,0x12c187931c00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16478,232_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op2,kernel_graph0,{17311},17311,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16985,16990,0x12c25ee10c00,0,DefaultEnhancedAscendMemoryPool,16778752,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16478,232_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16991,16996,0x12c20120ee00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16481,233_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op2,kernel_graph0,{16994},16994,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16994,17079,0x12c20520f000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16481,234_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/AddN-op0,kernel_graph0,{16997-17000-17008-17076},17076,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 16999,17006,0x12c23d610600,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16483,236_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op20,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op20,kernel_graph0,{17004},17004,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17000,17003,0x12c242c10800,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16483,236_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op20,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op20,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17004,17274,0x12c1df00e000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16484,237_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op20,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op20,kernel_graph0,{17272},17272,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17007,17020,0x12c24ee10800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16485,238_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op8,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op8,kernel_graph0,{17011-17013-17017},17017,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17008,17011,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16485,238_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op8,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op8,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17013,17036,0x12c20f20f600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16487,240_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op6,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op6,kernel_graph0,{17015-17027-17034},17034,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17017,17024,0x12c1d440de00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16488,242_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op7,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op7,kernel_graph0,{17020-17022},17022,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17022,17051,0x12c1b960d800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16490,244_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/Grad_SiLU/SiLUGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/Grad_SiLU/SiLUGrad-op1,kernel_graph0,{17041-17049},17049,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17026,17032,0x12c1e9c0e200,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16492,245_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op21,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op21,kernel_graph0,{17030},17030,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17027,17029,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16492,245_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op21,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op21,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17030,17277,0x12c24ee10800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16493,246_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op21,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op21,kernel_graph0,{17275},17275,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17033,17057,0x12c20120ee00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16493,247_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op9,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op9,kernel_graph0,{17038-17055},17055,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17034,17038,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16493,247_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op9,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op9,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17040,17047,0x12c1b400d600,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16497,249_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op19,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op19,kernel_graph0,{17045},17045,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17041,17044,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16497,249_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op19,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op19,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17045,17271,0x12c1c980dc00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16497,250_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op19,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op19,kernel_graph0,{17269},17269,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17048,17058,0x12c1a400d400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16499,251_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op10,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op10,kernel_graph0,{17053-17055},17055,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17049,17053,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16499,251_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op10,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op10,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17055,17061,0x12c1e9c0e200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16500,253_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/AddN-op0,kernel_graph0,{17059},17059,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17059,17069,0x12c1f320e600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16501,254_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op8,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op8,kernel_graph0,{17067},17067,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17062,17070,0x12c1d440de00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16502,255_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op2,kernel_graph0,{17067},17067,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17065,17075,0x12c1a400d400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16502,256_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op3,kernel_graph0,{17073},17073,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17066,17304,0x12c187929a00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16502,256_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op3,kernel_graph0,{17302},17302,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17067,17072,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,16778752,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16502,256_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op3,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17073,17078,0x12c20120ee00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16504,257_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op3,kernel_graph0,{17076},17076,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17076,17241,0x12c1e9c0e200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16505,258_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/AddN-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/AddN-op1,kernel_graph0,{17079-17082-17090-17238},17238,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17081,17088,0x12c259a10a00,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16507,260_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op18,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op18,kernel_graph0,{17086},17086,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17082,17085,0x12c1edc0e400,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16507,260_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op18,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op18,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17086,17268,0x12c25ba10c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16508,261_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op18,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op18,kernel_graph0,{17266},17266,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17089,17098,0x12c23d610600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16508,262_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op11,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op11,kernel_graph0,{17093-17096},17096,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17090,17093,0x12c1edc0e400,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16508,262_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op11,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op11,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17095,17109,0x12c1f320e600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16512,264_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op4,kernel_graph0,{17104},17104,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17096,17099,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16512,264_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op4,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17100,17122,0x12c1f720e800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16512,265_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,kernel_graph0,{17120},17120,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17101,17119,0x12c23d610600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16512,265_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,kernel_graph0,{17117},17117,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17102,17126,0x12c20120ee00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16512,265_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,kernel_graph0,{17124},17124,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17103,17115,0x12c180008800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16512,265_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17104,17116,0x12c20f20f600,0,DefaultEnhancedAscendMemoryPool,515969024,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16512,265_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17117,17133,0x12c18e00ca00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16514,266_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op11,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op11,kernel_graph0,{17129-17131},17131,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17120,17156,0x12c1d440de00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16515,267_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op9,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op9,kernel_graph0,{17150-17153},17153,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17123,17183,0x12c20520f000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16516,268_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op7,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op7,kernel_graph0,{17127-17174-17181},17181,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17124,17127,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16516,268_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op7,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op7,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17129,17139,0x12c1f320e600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16517,270_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op10,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op10,kernel_graph0,{17137},17137,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17131,17136,0x12c20f20f600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16518,271_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op11,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op11,kernel_graph0,{17134},17134,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17134,17140,0x12c18e00ca00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16519,272_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op3,kernel_graph0,{17137},17137,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17137,17143,0x12c20f20f600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16520,273_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op1,kernel_graph0,{17141},17141,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17141,17147,0x12c20120ee00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16520,274_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op12,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op12,kernel_graph0,{17145},17145,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17144,17197,0x12c23d610600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16522,275_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op6,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op6,kernel_graph0,{17148-17188-17195},17195,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17145,17148,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16522,275_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op6,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op6,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17150,17162,0x12c18e00ca00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16523,277_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op8,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op8,kernel_graph0,{17160},17160,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17153,17159,0x12c1f320e600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16526,278_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op9,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op9,kernel_graph0,{17157},17157,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17157,17163,0x12c1d440de00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16527,279_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op2,kernel_graph0,{17160},17160,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17160,17166,0x12c1f320e600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16527,280_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op0,kernel_graph0,{17164},17164,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17164,17170,0x12c20120ee00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16529,281_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op10,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op10,kernel_graph0,{17168},17168,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17167,17212,0x12c18e00ca00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16530,282_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op5,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op5,kernel_graph0,{17171-17202-17210},17210,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17168,17171,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16530,282_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op5,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op5,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17173,17179,0x12c20920f200,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16532,284_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op17,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op17,kernel_graph0,{17177},17177,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17174,17176,0x12c1edc0e400,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16532,284_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op17,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op17,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17177,17265,0x12c20120ee00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16533,285_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op17,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op17,kernel_graph0,{17263},17263,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17180,17220,0x12c1f320e600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16534,286_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op14,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op14,kernel_graph0,{17185-17216},17216,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17181,17185,0x12c1edc0e400,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16534,286_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op14,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op14,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17187,17193,0x12c259a10a00,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16537,288_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op16,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op16,kernel_graph0,{17191},17191,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17188,17190,0x12c1edc0e400,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16537,288_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op16,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op16,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17191,17262,0x12c1f720e800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16538,289_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op16,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op16,kernel_graph0,{17260},17260,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17194,17219,0x12c20520f000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16538,290_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op13,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op13,kernel_graph0,{17199-17216},17216,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17195,17199,0x12c1edc0e400,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16538,290_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op13,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op13,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17201,17208,0x12c1fb20ea00,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16545,292_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op15,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op15,kernel_graph0,{17206},17206,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17202,17205,0x12c1edc0e400,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16545,292_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op15,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op15,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17206,17259,0x12c23d610600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16546,293_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op15,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op15,kernel_graph0,{17257},17257,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17209,17218,0x12c1800a0c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16546,294_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op12,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op12,kernel_graph0,{17214-17216},17216,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17210,17214,0x12c1edc0e400,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16546,294_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op12,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op12,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17216,17223,0x12c1d440de00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16549,296_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/AddN-op0,kernel_graph0,{17221},17221,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17221,17231,0x12c20f20f600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16550,297_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op13,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op13,kernel_graph0,{17229},17229,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17224,17232,0x12c1d440de00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16550,298_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op2,kernel_graph0,{17229},17229,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17227,17237,0x12c21720f800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16551,299_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op4,kernel_graph0,{17235},17235,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17228,17307,0x12c180014800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16551,299_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op4,kernel_graph0,{17305},17305,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17229,17234,0x12c1edc0e400,0,DefaultEnhancedAscendMemoryPool,16778752,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16551,299_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op4,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17235,17240,0x12c1f320e600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16552,300_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op4,kernel_graph0,{17238},17238,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17238,17244,0x12c20520f000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16553,301_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/AddN-op0,kernel_graph0,{17242},17242,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17242,17247,0x12c1d440de00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16554,302_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op14,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op14,kernel_graph0,{17245},17245,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17245,17256,0x12c20f20f600,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16554,303_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/tok_embeddings-LlamaEmbedding/Grad_Gather/UnsortedSegmentSum-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/tok_embeddings-LlamaEmbedding/Grad_Gather/UnsortedSegmentSum-op0,kernel_graph0,{17254},17254,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17249,17253,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16556,304_Default/Mul-op1,Default/Mul-op1,kernel_graph0,{17251},17251,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17251,17317,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16557,305_Default/Reciprocal-op0,Default/Reciprocal-op0,kernel_graph0,{17254-17257-17260-17263-17266-17269-17272-17275-17278-17281-17284-17287-17290-17293-17296-17299-17302-17305-17308-17311-17314},17314,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17254,17552,0x12c187935e00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16557,306_Default/Mul-op2,Default/Mul-op2,kernel_graph0,{17318-17478-17550},17550,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17257,17555,0x12c1f320e600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16558,307_Default/Mul-op3,Default/Mul-op3,kernel_graph0,{17326-17478-17553},17553,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17260,17558,0x12c1e9c0e200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16559,308_Default/Mul-op4,Default/Mul-op4,kernel_graph0,{17334-17478-17556},17556,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17263,17561,0x12c1f720e800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16560,309_Default/Mul-op5,Default/Mul-op5,kernel_graph0,{17342-17478-17559},17559,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17266,17564,0x12c23d610600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16561,310_Default/Mul-op6,Default/Mul-op6,kernel_graph0,{17350-17478-17562},17562,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17269,17567,0x12c1d440de00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16561,311_Default/Mul-op7,Default/Mul-op7,kernel_graph0,{17358-17478-17565},17565,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17272,17570,0x12c20f20f600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16563,312_Default/Mul-op8,Default/Mul-op8,kernel_graph0,{17366-17478-17568},17568,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17275,17573,0x12c1df00e000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16564,313_Default/Mul-op9,Default/Mul-op9,kernel_graph0,{17374-17478-17571},17571,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17278,17576,0x12c180018a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16564,314_Default/Mul-op10,Default/Mul-op10,kernel_graph0,{17382-17478-17574},17574,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17281,17579,0x12c1edc0e400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16567,315_Default/Mul-op11,Default/Mul-op11,kernel_graph0,{17390-17478-17577},17577,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17284,17582,0x12c239610400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16567,316_Default/Mul-op12,Default/Mul-op12,kernel_graph0,{17398-17478-17580},17580,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17287,17585,0x12c1fb20ea00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16568,317_Default/Mul-op13,Default/Mul-op13,kernel_graph0,{17406-17478-17583},17583,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17290,17588,0x12c1ff20ec00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16568,318_Default/Mul-op14,Default/Mul-op14,kernel_graph0,{17414-17478-17586},17586,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17293,17591,0x12c219e0f800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16571,319_Default/Mul-op15,Default/Mul-op15,kernel_graph0,{17422-17478-17589},17589,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17296,17594,0x12c241610800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16572,320_Default/Mul-op16,Default/Mul-op16,kernel_graph0,{17430-17478-17592},17592,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17299,17597,0x12c24c210a00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16572,321_Default/Mul-op17,Default/Mul-op17,kernel_graph0,{17438-17478-17595},17595,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17302,17600,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16573,322_Default/Mul-op18,Default/Mul-op18,kernel_graph0,{17446-17478-17598},17598,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17305,17603,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16574,323_Default/Mul-op19,Default/Mul-op19,kernel_graph0,{17454-17478-17601},17601,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17308,17606,0x12c180014800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16575,324_Default/Mul-op20,Default/Mul-op20,kernel_graph0,{17462-17478-17604},17604,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17311,17609,0x12c180010600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16576,325_Default/Mul-op21,Default/Mul-op21,kernel_graph0,{17470-17478-17607},17607,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17314,17612,0x12c1f1c0e600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16576,326_Default/Mul-op22,Default/Mul-op22,kernel_graph0,{17478-17480-17610},17610,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17318,17323,0x12c1a6d36000,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16578,327_Default/clip_grad_norm-ClipGradNorm/Square-op0,Default/clip_grad_norm-ClipGradNorm/Square-op0,kernel_graph0,{17321},17321,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17320,17490,0x12c180008a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16579,328_Default/clip_grad_norm-ClipGradNorm/SumExt-op0,Default/clip_grad_norm-ClipGradNorm/SumExt-op0,kernel_graph0,{17324-17488},17488,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17321,17324,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16579,328_Default/clip_grad_norm-ClipGradNorm/SumExt-op0,Default/clip_grad_norm-ClipGradNorm/SumExt-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17326,17331,0x12c209e0ee00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16580,330_Default/clip_grad_norm-ClipGradNorm/Square-op1,Default/clip_grad_norm-ClipGradNorm/Square-op1,kernel_graph0,{17329},17329,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17328,17491,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16581,331_Default/clip_grad_norm-ClipGradNorm/SumExt-op1,Default/clip_grad_norm-ClipGradNorm/SumExt-op1,kernel_graph0,{17332-17488},17488,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17329,17332,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16581,331_Default/clip_grad_norm-ClipGradNorm/SumExt-op1,Default/clip_grad_norm-ClipGradNorm/SumExt-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17334,17339,0x12c209e0ee00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16583,333_Default/clip_grad_norm-ClipGradNorm/Square-op2,Default/clip_grad_norm-ClipGradNorm/Square-op2,kernel_graph0,{17337},17337,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17336,17492,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16583,334_Default/clip_grad_norm-ClipGradNorm/SumExt-op2,Default/clip_grad_norm-ClipGradNorm/SumExt-op2,kernel_graph0,{17340-17488},17488,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17337,17340,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16583,334_Default/clip_grad_norm-ClipGradNorm/SumExt-op2,Default/clip_grad_norm-ClipGradNorm/SumExt-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17342,17347,0x12c209e0ee00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16586,336_Default/clip_grad_norm-ClipGradNorm/Square-op3,Default/clip_grad_norm-ClipGradNorm/Square-op3,kernel_graph0,{17345},17345,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17344,17493,0x12c180009a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16586,337_Default/clip_grad_norm-ClipGradNorm/SumExt-op3,Default/clip_grad_norm-ClipGradNorm/SumExt-op3,kernel_graph0,{17348-17488},17488,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17345,17348,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16586,337_Default/clip_grad_norm-ClipGradNorm/SumExt-op3,Default/clip_grad_norm-ClipGradNorm/SumExt-op3,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17350,17355,0x12c209e0ee00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16589,339_Default/clip_grad_norm-ClipGradNorm/Square-op4,Default/clip_grad_norm-ClipGradNorm/Square-op4,kernel_graph0,{17353},17353,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17352,17494,0x12c180009c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16589,340_Default/clip_grad_norm-ClipGradNorm/SumExt-op4,Default/clip_grad_norm-ClipGradNorm/SumExt-op4,kernel_graph0,{17356-17488},17488,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17353,17356,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16589,340_Default/clip_grad_norm-ClipGradNorm/SumExt-op4,Default/clip_grad_norm-ClipGradNorm/SumExt-op4,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17358,17363,0x12c224a0fa00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16590,342_Default/clip_grad_norm-ClipGradNorm/Square-op5,Default/clip_grad_norm-ClipGradNorm/Square-op5,kernel_graph0,{17361},17361,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17360,17495,0x12c180009e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16591,343_Default/clip_grad_norm-ClipGradNorm/SumExt-op5,Default/clip_grad_norm-ClipGradNorm/SumExt-op5,kernel_graph0,{17364-17488},17488,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17361,17364,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16591,343_Default/clip_grad_norm-ClipGradNorm/SumExt-op5,Default/clip_grad_norm-ClipGradNorm/SumExt-op5,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17366,17371,0x12c224a0fa00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16592,345_Default/clip_grad_norm-ClipGradNorm/Square-op6,Default/clip_grad_norm-ClipGradNorm/Square-op6,kernel_graph0,{17369},17369,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17368,17496,0x12c18000a000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16593,346_Default/clip_grad_norm-ClipGradNorm/SumExt-op6,Default/clip_grad_norm-ClipGradNorm/SumExt-op6,kernel_graph0,{17372-17488},17488,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17369,17372,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16593,346_Default/clip_grad_norm-ClipGradNorm/SumExt-op6,Default/clip_grad_norm-ClipGradNorm/SumExt-op6,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17374,17379,0x12c224a0fa00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16595,348_Default/clip_grad_norm-ClipGradNorm/Square-op7,Default/clip_grad_norm-ClipGradNorm/Square-op7,kernel_graph0,{17377},17377,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17376,17497,0x12c18000a200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16595,349_Default/clip_grad_norm-ClipGradNorm/SumExt-op7,Default/clip_grad_norm-ClipGradNorm/SumExt-op7,kernel_graph0,{17380-17488},17488,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17377,17380,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16595,349_Default/clip_grad_norm-ClipGradNorm/SumExt-op7,Default/clip_grad_norm-ClipGradNorm/SumExt-op7,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17382,17387,0x12c209e0ee00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16598,351_Default/clip_grad_norm-ClipGradNorm/Square-op8,Default/clip_grad_norm-ClipGradNorm/Square-op8,kernel_graph0,{17385},17385,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17384,17498,0x12c18000a400,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16598,352_Default/clip_grad_norm-ClipGradNorm/SumExt-op8,Default/clip_grad_norm-ClipGradNorm/SumExt-op8,kernel_graph0,{17388-17488},17488,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17385,17388,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16598,352_Default/clip_grad_norm-ClipGradNorm/SumExt-op8,Default/clip_grad_norm-ClipGradNorm/SumExt-op8,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17390,17395,0x12c209e0ee00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16600,354_Default/clip_grad_norm-ClipGradNorm/Square-op9,Default/clip_grad_norm-ClipGradNorm/Square-op9,kernel_graph0,{17393},17393,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17392,17499,0x12c18000a600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16601,355_Default/clip_grad_norm-ClipGradNorm/SumExt-op9,Default/clip_grad_norm-ClipGradNorm/SumExt-op9,kernel_graph0,{17396-17488},17488,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17393,17396,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16601,355_Default/clip_grad_norm-ClipGradNorm/SumExt-op9,Default/clip_grad_norm-ClipGradNorm/SumExt-op9,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17398,17403,0x12c209e0ee00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16602,357_Default/clip_grad_norm-ClipGradNorm/Square-op10,Default/clip_grad_norm-ClipGradNorm/Square-op10,kernel_graph0,{17401},17401,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17400,17500,0x12c18000a800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16604,358_Default/clip_grad_norm-ClipGradNorm/SumExt-op10,Default/clip_grad_norm-ClipGradNorm/SumExt-op10,kernel_graph0,{17404-17488},17488,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17401,17404,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16604,358_Default/clip_grad_norm-ClipGradNorm/SumExt-op10,Default/clip_grad_norm-ClipGradNorm/SumExt-op10,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17406,17411,0x12c209e0ee00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16605,360_Default/clip_grad_norm-ClipGradNorm/Square-op11,Default/clip_grad_norm-ClipGradNorm/Square-op11,kernel_graph0,{17409},17409,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17408,17501,0x12c18000aa00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16608,361_Default/clip_grad_norm-ClipGradNorm/SumExt-op11,Default/clip_grad_norm-ClipGradNorm/SumExt-op11,kernel_graph0,{17412-17488},17488,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17409,17412,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16608,361_Default/clip_grad_norm-ClipGradNorm/SumExt-op11,Default/clip_grad_norm-ClipGradNorm/SumExt-op11,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17414,17419,0x12c224a0fa00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16609,363_Default/clip_grad_norm-ClipGradNorm/Square-op12,Default/clip_grad_norm-ClipGradNorm/Square-op12,kernel_graph0,{17417},17417,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17416,17502,0x12c18000ac00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16612,364_Default/clip_grad_norm-ClipGradNorm/SumExt-op12,Default/clip_grad_norm-ClipGradNorm/SumExt-op12,kernel_graph0,{17420-17488},17488,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17417,17420,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16612,364_Default/clip_grad_norm-ClipGradNorm/SumExt-op12,Default/clip_grad_norm-ClipGradNorm/SumExt-op12,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17422,17427,0x12c224a0fa00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16613,366_Default/clip_grad_norm-ClipGradNorm/Square-op13,Default/clip_grad_norm-ClipGradNorm/Square-op13,kernel_graph0,{17425},17425,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17424,17503,0x12c18000ae00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16614,367_Default/clip_grad_norm-ClipGradNorm/SumExt-op13,Default/clip_grad_norm-ClipGradNorm/SumExt-op13,kernel_graph0,{17428-17488},17488,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17425,17428,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16614,367_Default/clip_grad_norm-ClipGradNorm/SumExt-op13,Default/clip_grad_norm-ClipGradNorm/SumExt-op13,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17430,17435,0x12c224a0fa00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16616,369_Default/clip_grad_norm-ClipGradNorm/Square-op14,Default/clip_grad_norm-ClipGradNorm/Square-op14,kernel_graph0,{17433},17433,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17432,17504,0x12c18000b000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16617,370_Default/clip_grad_norm-ClipGradNorm/SumExt-op14,Default/clip_grad_norm-ClipGradNorm/SumExt-op14,kernel_graph0,{17436-17488},17488,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17433,17436,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16617,370_Default/clip_grad_norm-ClipGradNorm/SumExt-op14,Default/clip_grad_norm-ClipGradNorm/SumExt-op14,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17438,17443,0x12c1a6d36000,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16619,372_Default/clip_grad_norm-ClipGradNorm/Square-op15,Default/clip_grad_norm-ClipGradNorm/Square-op15,kernel_graph0,{17441},17441,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17440,17505,0x12c18000b200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16620,373_Default/clip_grad_norm-ClipGradNorm/SumExt-op15,Default/clip_grad_norm-ClipGradNorm/SumExt-op15,kernel_graph0,{17444-17488},17488,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17441,17444,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16620,373_Default/clip_grad_norm-ClipGradNorm/SumExt-op15,Default/clip_grad_norm-ClipGradNorm/SumExt-op15,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17446,17451,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16622,375_Default/clip_grad_norm-ClipGradNorm/Square-op16,Default/clip_grad_norm-ClipGradNorm/Square-op16,kernel_graph0,{17449},17449,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17448,17506,0x12c18000b400,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16623,376_Default/clip_grad_norm-ClipGradNorm/SumExt-op16,Default/clip_grad_norm-ClipGradNorm/SumExt-op16,kernel_graph0,{17452-17488},17488,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17449,17452,0x12c18000b600,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16623,376_Default/clip_grad_norm-ClipGradNorm/SumExt-op16,Default/clip_grad_norm-ClipGradNorm/SumExt-op16,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17454,17459,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16626,378_Default/clip_grad_norm-ClipGradNorm/Square-op17,Default/clip_grad_norm-ClipGradNorm/Square-op17,kernel_graph0,{17457},17457,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17456,17507,0x12c18000b600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16627,379_Default/clip_grad_norm-ClipGradNorm/SumExt-op17,Default/clip_grad_norm-ClipGradNorm/SumExt-op17,kernel_graph0,{17460-17488},17488,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17457,17460,0x12c18000b800,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16627,379_Default/clip_grad_norm-ClipGradNorm/SumExt-op17,Default/clip_grad_norm-ClipGradNorm/SumExt-op17,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17462,17467,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16629,381_Default/clip_grad_norm-ClipGradNorm/Square-op18,Default/clip_grad_norm-ClipGradNorm/Square-op18,kernel_graph0,{17465},17465,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17464,17508,0x12c18000b800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16630,382_Default/clip_grad_norm-ClipGradNorm/SumExt-op18,Default/clip_grad_norm-ClipGradNorm/SumExt-op18,kernel_graph0,{17468-17488},17488,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17465,17468,0x12c18000ba00,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16630,382_Default/clip_grad_norm-ClipGradNorm/SumExt-op18,Default/clip_grad_norm-ClipGradNorm/SumExt-op18,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17470,17475,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16633,384_Default/clip_grad_norm-ClipGradNorm/Square-op19,Default/clip_grad_norm-ClipGradNorm/Square-op19,kernel_graph0,{17473},17473,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17472,17509,0x12c18000ba00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16634,385_Default/clip_grad_norm-ClipGradNorm/SumExt-op19,Default/clip_grad_norm-ClipGradNorm/SumExt-op19,kernel_graph0,{17476-17488},17488,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17473,17476,0x12c1f1c12800,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16634,385_Default/clip_grad_norm-ClipGradNorm/SumExt-op19,Default/clip_grad_norm-ClipGradNorm/SumExt-op19,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17478,20843,0x12c18000bc00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16635,387_Default/AllFinite-op0,Default/AllFinite-op0,kernel_graph0,{17514-17517-17519},17519,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17480,17485,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16636,388_Default/clip_grad_norm-ClipGradNorm/Square-op20,Default/clip_grad_norm-ClipGradNorm/Square-op20,kernel_graph0,{17483},17483,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17482,17510,0x12c18000be00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16636,389_Default/clip_grad_norm-ClipGradNorm/SumExt-op20,Default/clip_grad_norm-ClipGradNorm/SumExt-op20,kernel_graph0,{17486-17488},17488,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17483,17486,0x12c1f1c12800,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16636,389_Default/clip_grad_norm-ClipGradNorm/SumExt-op20,Default/clip_grad_norm-ClipGradNorm/SumExt-op20,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17488,17513,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16638,391_Default/clip_grad_norm-ClipGradNorm/AddN-op0,Default/clip_grad_norm-ClipGradNorm/AddN-op0,kernel_graph0,{17511},17511,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17511,20840,0x12c180008a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16638,392_Default/clip_grad_norm-ClipGradNorm/Sqrt-op0,Default/clip_grad_norm-ClipGradNorm/Sqrt-op0,kernel_graph0,{17521-17523},17523,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17514,17544,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16638,393_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Select-op1,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Select-op1,kernel_graph0,{17535-17540},17540,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17517,17661,0x12c180008c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16639,394_Default/LogicalNot-op0,Default/LogicalNot-op0,kernel_graph0,{17537},17537,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17519,17547,0x12c18000be00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16639,395_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/LogicalOr-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/LogicalOr-op0,kernel_graph0,{17545},17545,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17521,17525,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16640,396_Default/clip_grad_norm-ClipGradNorm/GreaterEqual-op0,Default/clip_grad_norm-ClipGradNorm/GreaterEqual-op0,kernel_graph0,{17523},17523,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17523,17528,0x12c180009a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16640,397_Default/clip_grad_norm-ClipGradNorm/Select-op0,Default/clip_grad_norm-ClipGradNorm/Select-op0,kernel_graph0,{17526},17526,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17526,17531,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16644,398_Default/learning_rate-CosineWithWarmUpLR/Add-op0,Default/learning_rate-CosineWithWarmUpLR/Add-op0,kernel_graph0,{17529},17529,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17529,17534,0x12c180009a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16644,399_Default/clip_grad_norm-ClipGradNorm/Div-op0,Default/clip_grad_norm-ClipGradNorm/Div-op0,kernel_graph0,{17532},17532,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17532,17613,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16645,400_Default/clip_grad_norm-ClipGradNorm/Minimum-op0,Default/clip_grad_norm-ClipGradNorm/Minimum-op0,kernel_graph0,{17550-17553-17556-17559-17562-17565-17568-17571-17574-17577-17580-17583-17586-17589-17592-17595-17598-17601-17604-17607-17610},17610,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17535,17543,0x12c180009a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16645,401_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Mul-op1,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Mul-op1,kernel_graph0,{17540},17540,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17537,17542,0x12c180009c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16645,402_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/LogicalAnd-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/LogicalAnd-op0,kernel_graph0,{17540},17540,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17540,17619,0x12c180009600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16647,403_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Select-op2,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Select-op2,kernel_graph0,{17617},17617,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17545,17615,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16648,404_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Select-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Select-op0,kernel_graph0,{17613},17613,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17548,17617,0x12c18000be00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16649,405_Default/learning_rate-CosineWithWarmUpLR/Add-op1,Default/learning_rate-CosineWithWarmUpLR/Add-op1,kernel_graph0,{17615},17615,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17550,17820,0x12c1a6d36000,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16650,406_Default/clip_grad_norm-ClipGradNorm/Mul-op0,Default/clip_grad_norm-ClipGradNorm/Mul-op0,kernel_graph0,{17816-17818},17818,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17553,17810,0x12c209e0ee00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16651,407_Default/clip_grad_norm-ClipGradNorm/Mul-op1,Default/clip_grad_norm-ClipGradNorm/Mul-op1,kernel_graph0,{17806-17808},17808,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17556,17802,0x12c1f1c12800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16652,408_Default/clip_grad_norm-ClipGradNorm/Mul-op2,Default/clip_grad_norm-ClipGradNorm/Mul-op2,kernel_graph0,{17798-17800},17800,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17559,17794,0x12c1e9c0e200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16653,409_Default/clip_grad_norm-ClipGradNorm/Mul-op3,Default/clip_grad_norm-ClipGradNorm/Mul-op3,kernel_graph0,{17790-17792},17792,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17562,17786,0x12c1f5c12a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16654,410_Default/clip_grad_norm-ClipGradNorm/Mul-op4,Default/clip_grad_norm-ClipGradNorm/Mul-op4,kernel_graph0,{17782-17784},17784,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17565,17778,0x12c1c6136200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16655,411_Default/clip_grad_norm-ClipGradNorm/Mul-op5,Default/clip_grad_norm-ClipGradNorm/Mul-op5,kernel_graph0,{17774-17776},17776,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17568,17770,0x12c1d0d36400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16656,412_Default/clip_grad_norm-ClipGradNorm/Mul-op6,Default/clip_grad_norm-ClipGradNorm/Mul-op6,kernel_graph0,{17766-17768},17768,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17571,17762,0x12c20de0f000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16657,413_Default/clip_grad_norm-ClipGradNorm/Mul-op7,Default/clip_grad_norm-ClipGradNorm/Mul-op7,kernel_graph0,{17758-17760},17760,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17574,17754,0x12c23d610600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16658,414_Default/clip_grad_norm-ClipGradNorm/Mul-op8,Default/clip_grad_norm-ClipGradNorm/Mul-op8,kernel_graph0,{17750-17752},17752,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17577,17746,0x12c1db936600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16659,415_Default/clip_grad_norm-ClipGradNorm/Mul-op9,Default/clip_grad_norm-ClipGradNorm/Mul-op9,kernel_graph0,{17742-17744},17744,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17580,17738,0x12c1edc0e400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16660,416_Default/clip_grad_norm-ClipGradNorm/Mul-op10,Default/clip_grad_norm-ClipGradNorm/Mul-op10,kernel_graph0,{17734-17736},17736,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17583,17730,0x12c1df936800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16661,417_Default/clip_grad_norm-ClipGradNorm/Mul-op11,Default/clip_grad_norm-ClipGradNorm/Mul-op11,kernel_graph0,{17726-17728},17728,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17586,17722,0x12c224a0fa00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16661,418_Default/clip_grad_norm-ClipGradNorm/Mul-op12,Default/clip_grad_norm-ClipGradNorm/Mul-op12,kernel_graph0,{17718-17720},17720,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17589,17714,0x12c22f60fc00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16663,419_Default/clip_grad_norm-ClipGradNorm/Mul-op13,Default/clip_grad_norm-ClipGradNorm/Mul-op13,kernel_graph0,{17710-17712},17712,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17592,17706,0x12c218a0f200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16664,420_Default/clip_grad_norm-ClipGradNorm/Mul-op14,Default/clip_grad_norm-ClipGradNorm/Mul-op14,kernel_graph0,{17702-17704},17704,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17595,17698,0x12c180018a00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16664,421_Default/clip_grad_norm-ClipGradNorm/Mul-op15,Default/clip_grad_norm-ClipGradNorm/Mul-op15,kernel_graph0,{17694-17696},17696,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17598,17690,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16667,422_Default/clip_grad_norm-ClipGradNorm/Mul-op16,Default/clip_grad_norm-ClipGradNorm/Mul-op16,kernel_graph0,{17686-17688},17688,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17601,17682,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16668,423_Default/clip_grad_norm-ClipGradNorm/Mul-op17,Default/clip_grad_norm-ClipGradNorm/Mul-op17,kernel_graph0,{17678-17680},17680,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17604,17674,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16668,424_Default/clip_grad_norm-ClipGradNorm/Mul-op18,Default/clip_grad_norm-ClipGradNorm/Mul-op18,kernel_graph0,{17670-17672},17672,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17607,17666,0x12c180014800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16668,425_Default/clip_grad_norm-ClipGradNorm/Mul-op19,Default/clip_grad_norm-ClipGradNorm/Mul-op19,kernel_graph0,{17662-17664},17664,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17610,17828,0x12c180010600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16668,426_Default/clip_grad_norm-ClipGradNorm/Mul-op20,Default/clip_grad_norm-ClipGradNorm/Mul-op20,kernel_graph0,{17824-17826},17826,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17620,17629,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16671,430_Default/learning_rate-CosineWithWarmUpLR/Cast-op0,Default/learning_rate-CosineWithWarmUpLR/Cast-op0,kernel_graph0,{17622-17625-17627},17627,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17622,17624,0x12c18000be00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,16672,431_Default/learning_rate-CosineWithWarmUpLR/GreaterEqual-op0,Default/learning_rate-CosineWithWarmUpLR/GreaterEqual-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17625,17630,0x12c18000be00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17624,435_Default/learning_rate-CosineWithWarmUpLR/Greater-op0,Default/learning_rate-CosineWithWarmUpLR/Greater-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17627,17633,0x12c180009600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17625,436_Default/TensorMove-op0,Default/TensorMove-op0,kernel_graph0,{17631},17631,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17631,17636,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17630,440_Default/learning_rate-CosineWithWarmUpLR/Sub-op1,Default/learning_rate-CosineWithWarmUpLR/Sub-op1,kernel_graph0,{17634},17634,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17634,17639,0x12c18000be00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17631,441_Default/learning_rate-CosineWithWarmUpLR/Div-op1,Default/learning_rate-CosineWithWarmUpLR/Div-op1,kernel_graph0,{17637},17637,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17637,17642,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17632,442_Default/learning_rate-CosineWithWarmUpLR/Mul-op1,Default/learning_rate-CosineWithWarmUpLR/Mul-op1,kernel_graph0,{17640},17640,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17640,17645,0x12c18000be00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17632,443_Default/learning_rate-CosineWithWarmUpLR/Cos-op0,Default/learning_rate-CosineWithWarmUpLR/Cos-op0,kernel_graph0,{17643},17643,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17643,17648,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17634,444_Default/learning_rate-CosineWithWarmUpLR/Add-op3,Default/learning_rate-CosineWithWarmUpLR/Add-op3,kernel_graph0,{17646},17646,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17646,17651,0x12c18000be00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17635,445_Default/learning_rate-CosineWithWarmUpLR/Mul-op2,Default/learning_rate-CosineWithWarmUpLR/Mul-op2,kernel_graph0,{17649},17649,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17649,17654,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17635,446_Default/learning_rate-CosineWithWarmUpLR/Maximum-op0,Default/learning_rate-CosineWithWarmUpLR/Maximum-op0,kernel_graph0,{17652},17652,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17652,17657,0x12c18000be00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17637,447_Default/learning_rate-CosineWithWarmUpLR/Mul-op3,Default/learning_rate-CosineWithWarmUpLR/Mul-op3,kernel_graph0,{17655},17655,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17655,17660,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17638,448_Default/learning_rate-CosineWithWarmUpLR/Add-op4,Default/learning_rate-CosineWithWarmUpLR/Add-op4,kernel_graph0,{17658},17658,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17658,20841,0x12c18000be00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17641,451_Default/Reshape-op0,Default/Reshape-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17662,18491,0x12c1f1c0e600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17661,453_Default/optimizer-FP32StateAdamWeightDecay/Mul-op96,Default/optimizer-FP32StateAdamWeightDecay/Mul-op96,kernel_graph0,{18488},18488,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17664,17669,0x12c22360f400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17662,454_Default/optimizer-FP32StateAdamWeightDecay/Square-op19,Default/optimizer-FP32StateAdamWeightDecay/Square-op19,kernel_graph0,{17667},17667,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17667,18487,0x12c180014800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17663,455_Default/optimizer-FP32StateAdamWeightDecay/Mul-op98,Default/optimizer-FP32StateAdamWeightDecay/Mul-op98,kernel_graph0,{18484},18484,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17670,18455,0x12c22360f400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17663,456_Default/optimizer-FP32StateAdamWeightDecay/Mul-op91,Default/optimizer-FP32StateAdamWeightDecay/Mul-op91,kernel_graph0,{18452},18452,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17672,17677,0x12c223613600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17664,457_Default/optimizer-FP32StateAdamWeightDecay/Square-op18,Default/optimizer-FP32StateAdamWeightDecay/Square-op18,kernel_graph0,{17675},17675,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17675,18459,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17665,458_Default/optimizer-FP32StateAdamWeightDecay/Mul-op93,Default/optimizer-FP32StateAdamWeightDecay/Mul-op93,kernel_graph0,{18456},18456,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17678,18423,0x12c223613600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17665,459_Default/optimizer-FP32StateAdamWeightDecay/Mul-op86,Default/optimizer-FP32StateAdamWeightDecay/Mul-op86,kernel_graph0,{18420},18420,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17680,17685,0x12c223617800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17667,460_Default/optimizer-FP32StateAdamWeightDecay/Square-op17,Default/optimizer-FP32StateAdamWeightDecay/Square-op17,kernel_graph0,{17683},17683,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17683,18427,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17668,461_Default/optimizer-FP32StateAdamWeightDecay/Mul-op88,Default/optimizer-FP32StateAdamWeightDecay/Mul-op88,kernel_graph0,{18424},18424,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17686,18391,0x12c223617800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17668,462_Default/optimizer-FP32StateAdamWeightDecay/Mul-op81,Default/optimizer-FP32StateAdamWeightDecay/Mul-op81,kernel_graph0,{18388},18388,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17688,17693,0x12c22361ba00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17670,463_Default/optimizer-FP32StateAdamWeightDecay/Square-op16,Default/optimizer-FP32StateAdamWeightDecay/Square-op16,kernel_graph0,{17691},17691,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17691,18395,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17671,464_Default/optimizer-FP32StateAdamWeightDecay/Mul-op83,Default/optimizer-FP32StateAdamWeightDecay/Mul-op83,kernel_graph0,{18392},18392,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17694,18359,0x12c241610800,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17671,465_Default/optimizer-FP32StateAdamWeightDecay/Mul-op76,Default/optimizer-FP32StateAdamWeightDecay/Mul-op76,kernel_graph0,{18356},18356,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17696,17701,0x12c260a10a00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17672,466_Default/optimizer-FP32StateAdamWeightDecay/Square-op15,Default/optimizer-FP32StateAdamWeightDecay/Square-op15,kernel_graph0,{17699},17699,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17699,18363,0x12c180018a00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17673,467_Default/optimizer-FP32StateAdamWeightDecay/Mul-op78,Default/optimizer-FP32StateAdamWeightDecay/Mul-op78,kernel_graph0,{18360},18360,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17702,18327,0x12c1f9c12c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17673,468_Default/optimizer-FP32StateAdamWeightDecay/Mul-op71,Default/optimizer-FP32StateAdamWeightDecay/Mul-op71,kernel_graph0,{18324},18324,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17704,17709,0x12c260a10a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17675,469_Default/optimizer-FP32StateAdamWeightDecay/Square-op14,Default/optimizer-FP32StateAdamWeightDecay/Square-op14,kernel_graph0,{17707},17707,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17707,18331,0x12c218a0f200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17676,470_Default/optimizer-FP32StateAdamWeightDecay/Mul-op73,Default/optimizer-FP32StateAdamWeightDecay/Mul-op73,kernel_graph0,{18328},18328,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17710,18295,0x12c260a10a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17676,471_Default/optimizer-FP32StateAdamWeightDecay/Mul-op66,Default/optimizer-FP32StateAdamWeightDecay/Mul-op66,kernel_graph0,{18292},18292,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17712,17717,0x12c26b610c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17678,472_Default/optimizer-FP32StateAdamWeightDecay/Square-op13,Default/optimizer-FP32StateAdamWeightDecay/Square-op13,kernel_graph0,{17715},17715,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17715,18299,0x12c22f60fc00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17679,473_Default/optimizer-FP32StateAdamWeightDecay/Mul-op68,Default/optimizer-FP32StateAdamWeightDecay/Mul-op68,kernel_graph0,{18296},18296,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17718,18263,0x12c26b610c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17679,474_Default/optimizer-FP32StateAdamWeightDecay/Mul-op61,Default/optimizer-FP32StateAdamWeightDecay/Mul-op61,kernel_graph0,{18260},18260,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17720,17725,0x12c276210e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17680,475_Default/optimizer-FP32StateAdamWeightDecay/Square-op12,Default/optimizer-FP32StateAdamWeightDecay/Square-op12,kernel_graph0,{17723},17723,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17723,18267,0x12c22361ba00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17681,476_Default/optimizer-FP32StateAdamWeightDecay/Mul-op63,Default/optimizer-FP32StateAdamWeightDecay/Mul-op63,kernel_graph0,{18264},18264,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17726,18231,0x12c204812e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17681,477_Default/optimizer-FP32StateAdamWeightDecay/Mul-op56,Default/optimizer-FP32StateAdamWeightDecay/Mul-op56,kernel_graph0,{18228},18228,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17728,17733,0x12c1e3936a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17683,478_Default/optimizer-FP32StateAdamWeightDecay/Square-op11,Default/optimizer-FP32StateAdamWeightDecay/Square-op11,kernel_graph0,{17731},17731,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17731,18235,0x12c1df936800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17684,479_Default/optimizer-FP32StateAdamWeightDecay/Mul-op58,Default/optimizer-FP32StateAdamWeightDecay/Mul-op58,kernel_graph0,{18232},18232,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17734,18199,0x12c1e3936a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17684,480_Default/optimizer-FP32StateAdamWeightDecay/Mul-op51,Default/optimizer-FP32StateAdamWeightDecay/Mul-op51,kernel_graph0,{18196},18196,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17736,17741,0x12c19f418c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17686,481_Default/optimizer-FP32StateAdamWeightDecay/Square-op10,Default/optimizer-FP32StateAdamWeightDecay/Square-op10,kernel_graph0,{17739},17739,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17739,18203,0x12c1edc0e400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17687,482_Default/optimizer-FP32StateAdamWeightDecay/Mul-op53,Default/optimizer-FP32StateAdamWeightDecay/Mul-op53,kernel_graph0,{18200},18200,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17742,18167,0x12c19f418c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17687,483_Default/optimizer-FP32StateAdamWeightDecay/Mul-op46,Default/optimizer-FP32StateAdamWeightDecay/Mul-op46,kernel_graph0,{18164},18164,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17744,17749,0x12c276210e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17688,484_Default/optimizer-FP32StateAdamWeightDecay/Square-op9,Default/optimizer-FP32StateAdamWeightDecay/Square-op9,kernel_graph0,{17747},17747,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17747,18171,0x12c1db936600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17689,485_Default/optimizer-FP32StateAdamWeightDecay/Mul-op48,Default/optimizer-FP32StateAdamWeightDecay/Mul-op48,kernel_graph0,{18168},18168,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17750,18135,0x12c276210e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17689,486_Default/optimizer-FP32StateAdamWeightDecay/Mul-op41,Default/optimizer-FP32StateAdamWeightDecay/Mul-op41,kernel_graph0,{18132},18132,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17752,17757,0x12c27a211000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17691,487_Default/optimizer-FP32StateAdamWeightDecay/Square-op8,Default/optimizer-FP32StateAdamWeightDecay/Square-op8,kernel_graph0,{17755},17755,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17755,18139,0x12c23a20fe00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17692,488_Default/optimizer-FP32StateAdamWeightDecay/Mul-op43,Default/optimizer-FP32StateAdamWeightDecay/Mul-op43,kernel_graph0,{18136},18136,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17758,18103,0x12c27a211000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17692,489_Default/optimizer-FP32StateAdamWeightDecay/Mul-op36,Default/optimizer-FP32StateAdamWeightDecay/Mul-op36,kernel_graph0,{18100},18100,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17760,17765,0x12c284e11200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17694,490_Default/optimizer-FP32StateAdamWeightDecay/Square-op7,Default/optimizer-FP32StateAdamWeightDecay/Square-op7,kernel_graph0,{17763},17763,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17763,18107,0x12c20de0f000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17695,491_Default/optimizer-FP32StateAdamWeightDecay/Mul-op38,Default/optimizer-FP32StateAdamWeightDecay/Mul-op38,kernel_graph0,{18104},18104,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17766,18071,0x12c284e11200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17696,492_Default/optimizer-FP32StateAdamWeightDecay/Mul-op31,Default/optimizer-FP32StateAdamWeightDecay/Mul-op31,kernel_graph0,{18068},18068,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17768,17773,0x12c28fa11400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17697,493_Default/optimizer-FP32StateAdamWeightDecay/Square-op6,Default/optimizer-FP32StateAdamWeightDecay/Square-op6,kernel_graph0,{17771},17771,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17771,18075,0x12c1d0d36400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17697,494_Default/optimizer-FP32StateAdamWeightDecay/Mul-op33,Default/optimizer-FP32StateAdamWeightDecay/Mul-op33,kernel_graph0,{18072},18072,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17774,18039,0x12c28fa11400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17699,495_Default/optimizer-FP32StateAdamWeightDecay/Mul-op26,Default/optimizer-FP32StateAdamWeightDecay/Mul-op26,kernel_graph0,{18036},18036,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17776,17781,0x12c29a611600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17700,496_Default/optimizer-FP32StateAdamWeightDecay/Square-op5,Default/optimizer-FP32StateAdamWeightDecay/Square-op5,kernel_graph0,{17779},17779,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17779,18043,0x12c1c6136200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17700,497_Default/optimizer-FP32StateAdamWeightDecay/Mul-op28,Default/optimizer-FP32StateAdamWeightDecay/Mul-op28,kernel_graph0,{18040},18040,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17782,18007,0x12c29a611600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17702,498_Default/optimizer-FP32StateAdamWeightDecay/Mul-op21,Default/optimizer-FP32StateAdamWeightDecay/Mul-op21,kernel_graph0,{18004},18004,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17784,17789,0x12c29e611800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17703,499_Default/optimizer-FP32StateAdamWeightDecay/Square-op4,Default/optimizer-FP32StateAdamWeightDecay/Square-op4,kernel_graph0,{17787},17787,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17787,18011,0x12c1f5c12a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17703,500_Default/optimizer-FP32StateAdamWeightDecay/Mul-op23,Default/optimizer-FP32StateAdamWeightDecay/Mul-op23,kernel_graph0,{18008},18008,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17790,17975,0x12c29e611800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17704,501_Default/optimizer-FP32StateAdamWeightDecay/Mul-op16,Default/optimizer-FP32StateAdamWeightDecay/Mul-op16,kernel_graph0,{17972},17972,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17792,17797,0x12c2a2611a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17705,502_Default/optimizer-FP32StateAdamWeightDecay/Square-op3,Default/optimizer-FP32StateAdamWeightDecay/Square-op3,kernel_graph0,{17795},17795,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17795,17979,0x12c1e7936c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17706,503_Default/optimizer-FP32StateAdamWeightDecay/Mul-op18,Default/optimizer-FP32StateAdamWeightDecay/Mul-op18,kernel_graph0,{17976},17976,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17798,17943,0x12c2a2611a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17707,504_Default/optimizer-FP32StateAdamWeightDecay/Mul-op11,Default/optimizer-FP32StateAdamWeightDecay/Mul-op11,kernel_graph0,{17940},17940,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17800,17805,0x12c2a6611c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17708,505_Default/optimizer-FP32StateAdamWeightDecay/Square-op2,Default/optimizer-FP32StateAdamWeightDecay/Square-op2,kernel_graph0,{17803},17803,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17803,17947,0x12c1f1c12800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17709,506_Default/optimizer-FP32StateAdamWeightDecay/Mul-op13,Default/optimizer-FP32StateAdamWeightDecay/Mul-op13,kernel_graph0,{17944},17944,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17806,17911,0x12c2a6611c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17710,507_Default/optimizer-FP32StateAdamWeightDecay/Mul-op6,Default/optimizer-FP32StateAdamWeightDecay/Mul-op6,kernel_graph0,{17908},17908,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17808,17813,0x12c2aa611e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17711,508_Default/optimizer-FP32StateAdamWeightDecay/Square-op1,Default/optimizer-FP32StateAdamWeightDecay/Square-op1,kernel_graph0,{17811},17811,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17811,17915,0x12c208813000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17711,509_Default/optimizer-FP32StateAdamWeightDecay/Mul-op8,Default/optimizer-FP32StateAdamWeightDecay/Mul-op8,kernel_graph0,{17912},17912,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17814,17839,0x12c180008c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17712,510_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Cast-op0,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Cast-op0,kernel_graph0,{17832-17835-17837},17837,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17816,17879,0x12c2aa611e00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17713,511_Default/optimizer-FP32StateAdamWeightDecay/Mul-op1,Default/optimizer-FP32StateAdamWeightDecay/Mul-op1,kernel_graph0,{17876},17876,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17818,17823,0x12c2c9a12000,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17713,512_Default/optimizer-FP32StateAdamWeightDecay/Square-op0,Default/optimizer-FP32StateAdamWeightDecay/Square-op0,kernel_graph0,{17821},17821,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17821,17883,0x12c1a3418e00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17715,513_Default/optimizer-FP32StateAdamWeightDecay/Mul-op3,Default/optimizer-FP32StateAdamWeightDecay/Mul-op3,kernel_graph0,{17880},17880,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17824,18523,0x12c22e21bc00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17716,514_Default/optimizer-FP32StateAdamWeightDecay/Mul-op101,Default/optimizer-FP32StateAdamWeightDecay/Mul-op101,kernel_graph0,{18520},18520,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17826,17831,0x12c22e21fe00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17716,515_Default/optimizer-FP32StateAdamWeightDecay/Square-op20,Default/optimizer-FP32StateAdamWeightDecay/Square-op20,kernel_graph0,{17829},17829,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17829,18519,0x12c180010600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17718,516_Default/optimizer-FP32StateAdamWeightDecay/Mul-op103,Default/optimizer-FP32StateAdamWeightDecay/Mul-op103,kernel_graph0,{18516},18516,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17832,17834,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17719,518_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/GreaterEqual-op0,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/GreaterEqual-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17835,17840,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17834,522_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Greater-op0,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Greater-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17837,17843,0x12c180009600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17835,523_Default/TensorMove-op1,Default/TensorMove-op1,kernel_graph0,{17841},17841,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17841,17846,0x12c180008c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17840,527_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Sub-op1,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Sub-op1,kernel_graph0,{17844},17844,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17844,17849,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17841,528_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Div-op1,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Div-op1,kernel_graph0,{17847},17847,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17847,17852,0x12c180008c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17842,529_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Mul-op1,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Mul-op1,kernel_graph0,{17850},17850,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17850,17855,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17842,530_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Cos-op0,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Cos-op0,kernel_graph0,{17853},17853,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17853,17858,0x12c180008c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17844,531_Default/learning_rate-CosineWithWarmUpLR/Add-op6,Default/learning_rate-CosineWithWarmUpLR/Add-op6,kernel_graph0,{17856},17856,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17856,17861,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17845,532_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Mul-op2,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Mul-op2,kernel_graph0,{17859},17859,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17859,17864,0x12c180008c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17845,533_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Maximum-op0,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Maximum-op0,kernel_graph0,{17862},17862,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17862,17867,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17847,534_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Mul-op3,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Mul-op3,kernel_graph0,{17865},17865,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17865,17870,0x12c180008c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17848,535_Default/learning_rate-CosineWithWarmUpLR/Add-op7,Default/learning_rate-CosineWithWarmUpLR/Add-op7,kernel_graph0,{17868},17868,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17868,18534,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17851,538_Default/optimizer-FP32StateAdamWeightDecay/Reshape-op0,Default/optimizer-FP32StateAdamWeightDecay/Reshape-op0,kernel_graph0,{17892-17924-17956-17988-18020-18052-18084-18116-18148-18180-18212-18244-18276-18308-18340-18372-18404-18436-18468-18500-18532},18532,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17872,17878,0x12c2c9a12000,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17853,540_Default/optimizer-FP32StateAdamWeightDecay/Mul-op0,Default/optimizer-FP32StateAdamWeightDecay/Mul-op0,kernel_graph0,{17876},17876,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17874,17882,0x12c2e8e12200,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17854,541_Default/optimizer-FP32StateAdamWeightDecay/Mul-op2,Default/optimizer-FP32StateAdamWeightDecay/Mul-op2,kernel_graph0,{17880},17880,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17876,17901,0x12c308212400,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17854,542_Default/learning_rate-CosineWithWarmUpLR/Add-op8,Default/learning_rate-CosineWithWarmUpLR/Add-op8,kernel_graph0,{17889-17899},17899,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17880,17899,0x12c2aa611e00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17856,543_Default/learning_rate-CosineWithWarmUpLR/Add-op9,Default/learning_rate-CosineWithWarmUpLR/Add-op9,kernel_graph0,{17884-17897},17897,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17884,17888,0x12c1a3418e00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17857,544_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op0,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op0,kernel_graph0,{17886},17886,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17886,17891,0x12c2c9a12000,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17857,545_Default/learning_rate-CosineWithWarmUpLR/Add-op10,Default/learning_rate-CosineWithWarmUpLR/Add-op10,kernel_graph0,{17889},17889,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17889,17894,0x12c2e8e12200,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17859,546_Default/optimizer-FP32StateAdamWeightDecay/Div-op0,Default/optimizer-FP32StateAdamWeightDecay/Div-op0,kernel_graph0,{17892},17892,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17892,17897,0x12c2c9a12000,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17860,547_Default/optimizer-FP32StateAdamWeightDecay/Mul-op4,Default/optimizer-FP32StateAdamWeightDecay/Mul-op4,kernel_graph0,{17895},17895,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17895,17903,0x12c2e8e12200,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17860,548_Default/optimizer-FP32StateAdamWeightDecay/Sub-op0,Default/optimizer-FP32StateAdamWeightDecay/Sub-op0,kernel_graph0,{17901},17901,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17904,17910,0x12c1a3418e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17865,552_Default/optimizer-FP32StateAdamWeightDecay/Mul-op5,Default/optimizer-FP32StateAdamWeightDecay/Mul-op5,kernel_graph0,{17908},17908,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17906,17914,0x12c1a7419000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17866,553_Default/optimizer-FP32StateAdamWeightDecay/Mul-op7,Default/optimizer-FP32StateAdamWeightDecay/Mul-op7,kernel_graph0,{17912},17912,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17908,17933,0x12c1ab419200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17866,554_Default/learning_rate-CosineWithWarmUpLR/Add-op11,Default/learning_rate-CosineWithWarmUpLR/Add-op11,kernel_graph0,{17921-17931},17931,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17912,17931,0x12c1a3418e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17868,555_Default/learning_rate-CosineWithWarmUpLR/Add-op12,Default/learning_rate-CosineWithWarmUpLR/Add-op12,kernel_graph0,{17916-17929},17929,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17916,17920,0x12c1a7419000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17869,556_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op1,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op1,kernel_graph0,{17918},17918,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17918,17923,0x12c208813000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17870,557_Default/learning_rate-CosineWithWarmUpLR/Add-op13,Default/learning_rate-CosineWithWarmUpLR/Add-op13,kernel_graph0,{17921},17921,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17921,17926,0x12c1a7419000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17870,558_Default/optimizer-FP32StateAdamWeightDecay/Div-op1,Default/optimizer-FP32StateAdamWeightDecay/Div-op1,kernel_graph0,{17924},17924,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17924,17929,0x12c208813000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17871,559_Default/optimizer-FP32StateAdamWeightDecay/Mul-op9,Default/optimizer-FP32StateAdamWeightDecay/Mul-op9,kernel_graph0,{17927},17927,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17927,17935,0x12c1a7419000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17871,560_Default/optimizer-FP32StateAdamWeightDecay/Sub-op1,Default/optimizer-FP32StateAdamWeightDecay/Sub-op1,kernel_graph0,{17933},17933,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17936,17942,0x12c208813000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17873,564_Default/optimizer-FP32StateAdamWeightDecay/Mul-op10,Default/optimizer-FP32StateAdamWeightDecay/Mul-op10,kernel_graph0,{17940},17940,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17938,17946,0x12c1a3418e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17873,565_Default/optimizer-FP32StateAdamWeightDecay/Mul-op12,Default/optimizer-FP32StateAdamWeightDecay/Mul-op12,kernel_graph0,{17944},17944,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17940,17965,0x12c1a7419000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17874,566_Default/learning_rate-CosineWithWarmUpLR/Add-op14,Default/learning_rate-CosineWithWarmUpLR/Add-op14,kernel_graph0,{17953-17963},17963,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17944,17963,0x12c208813000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17875,567_Default/learning_rate-CosineWithWarmUpLR/Add-op15,Default/learning_rate-CosineWithWarmUpLR/Add-op15,kernel_graph0,{17948-17961},17961,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17948,17952,0x12c1a3418e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17875,568_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op2,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op2,kernel_graph0,{17950},17950,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17950,17955,0x12c1f1c12800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17876,569_Default/learning_rate-CosineWithWarmUpLR/Add-op16,Default/learning_rate-CosineWithWarmUpLR/Add-op16,kernel_graph0,{17953},17953,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17953,17958,0x12c1a3418e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17877,570_Default/optimizer-FP32StateAdamWeightDecay/Div-op2,Default/optimizer-FP32StateAdamWeightDecay/Div-op2,kernel_graph0,{17956},17956,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17956,17961,0x12c1f1c12800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17877,571_Default/optimizer-FP32StateAdamWeightDecay/Mul-op14,Default/optimizer-FP32StateAdamWeightDecay/Mul-op14,kernel_graph0,{17959},17959,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17959,17967,0x12c1a3418e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17880,572_Default/optimizer-FP32StateAdamWeightDecay/Sub-op2,Default/optimizer-FP32StateAdamWeightDecay/Sub-op2,kernel_graph0,{17965},17965,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17968,17974,0x12c1f1c12800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17885,576_Default/optimizer-FP32StateAdamWeightDecay/Mul-op15,Default/optimizer-FP32StateAdamWeightDecay/Mul-op15,kernel_graph0,{17972},17972,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17970,17978,0x12c208813000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17885,577_Default/optimizer-FP32StateAdamWeightDecay/Mul-op17,Default/optimizer-FP32StateAdamWeightDecay/Mul-op17,kernel_graph0,{17976},17976,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17972,17997,0x12c1a3418e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17886,578_Default/learning_rate-CosineWithWarmUpLR/Add-op17,Default/learning_rate-CosineWithWarmUpLR/Add-op17,kernel_graph0,{17985-17995},17995,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17976,17995,0x12c1f1c12800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17887,579_Default/learning_rate-CosineWithWarmUpLR/Add-op18,Default/learning_rate-CosineWithWarmUpLR/Add-op18,kernel_graph0,{17980-17993},17993,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17980,17984,0x12c208813000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17887,580_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op3,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op3,kernel_graph0,{17982},17982,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17982,17987,0x12c1e7936c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17889,581_Default/learning_rate-CosineWithWarmUpLR/Add-op19,Default/learning_rate-CosineWithWarmUpLR/Add-op19,kernel_graph0,{17985},17985,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17985,17990,0x12c208813000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17890,582_Default/optimizer-FP32StateAdamWeightDecay/Div-op3,Default/optimizer-FP32StateAdamWeightDecay/Div-op3,kernel_graph0,{17988},17988,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17988,17993,0x12c1e7936c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17890,583_Default/optimizer-FP32StateAdamWeightDecay/Mul-op19,Default/optimizer-FP32StateAdamWeightDecay/Mul-op19,kernel_graph0,{17991},17991,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 17991,17999,0x12c208813000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17892,584_Default/optimizer-FP32StateAdamWeightDecay/Sub-op3,Default/optimizer-FP32StateAdamWeightDecay/Sub-op3,kernel_graph0,{17997},17997,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18000,18006,0x12c1f1c12800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17896,588_Default/optimizer-FP32StateAdamWeightDecay/Mul-op20,Default/optimizer-FP32StateAdamWeightDecay/Mul-op20,kernel_graph0,{18004},18004,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18002,18010,0x12c208813000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17896,589_Default/optimizer-FP32StateAdamWeightDecay/Mul-op22,Default/optimizer-FP32StateAdamWeightDecay/Mul-op22,kernel_graph0,{18008},18008,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18004,18029,0x12c1e7936c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17897,590_Default/learning_rate-CosineWithWarmUpLR/Add-op20,Default/learning_rate-CosineWithWarmUpLR/Add-op20,kernel_graph0,{18017-18027},18027,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18008,18027,0x12c1f1c12800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17898,591_Default/learning_rate-CosineWithWarmUpLR/Add-op21,Default/learning_rate-CosineWithWarmUpLR/Add-op21,kernel_graph0,{18012-18025},18025,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18012,18016,0x12c1f5c12a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17898,592_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op4,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op4,kernel_graph0,{18014},18014,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18014,18019,0x12c208813000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17898,593_Default/learning_rate-CosineWithWarmUpLR/Add-op22,Default/learning_rate-CosineWithWarmUpLR/Add-op22,kernel_graph0,{18017},18017,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18017,18022,0x12c1f5c12a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17898,594_Default/optimizer-FP32StateAdamWeightDecay/Div-op4,Default/optimizer-FP32StateAdamWeightDecay/Div-op4,kernel_graph0,{18020},18020,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18020,18025,0x12c208813000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17899,595_Default/optimizer-FP32StateAdamWeightDecay/Mul-op24,Default/optimizer-FP32StateAdamWeightDecay/Mul-op24,kernel_graph0,{18023},18023,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18023,18031,0x12c1f5c12a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17900,596_Default/optimizer-FP32StateAdamWeightDecay/Sub-op4,Default/optimizer-FP32StateAdamWeightDecay/Sub-op4,kernel_graph0,{18029},18029,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18032,18038,0x12c1a3418e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17902,600_Default/optimizer-FP32StateAdamWeightDecay/Mul-op25,Default/optimizer-FP32StateAdamWeightDecay/Mul-op25,kernel_graph0,{18036},18036,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18034,18042,0x12c1ae019000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17902,601_Default/optimizer-FP32StateAdamWeightDecay/Mul-op27,Default/optimizer-FP32StateAdamWeightDecay/Mul-op27,kernel_graph0,{18040},18040,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18036,18061,0x12c1b8c19200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17902,602_Default/learning_rate-CosineWithWarmUpLR/Add-op23,Default/learning_rate-CosineWithWarmUpLR/Add-op23,kernel_graph0,{18049-18059},18059,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18040,18059,0x12c1a3418e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17903,603_Default/learning_rate-CosineWithWarmUpLR/Add-op24,Default/learning_rate-CosineWithWarmUpLR/Add-op24,kernel_graph0,{18044-18057},18057,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18044,18048,0x12c1ae019000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17904,604_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op5,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op5,kernel_graph0,{18046},18046,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18046,18051,0x12c1c3819400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17905,605_Default/learning_rate-CosineWithWarmUpLR/Add-op25,Default/learning_rate-CosineWithWarmUpLR/Add-op25,kernel_graph0,{18049},18049,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18049,18054,0x12c1ae019000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17905,606_Default/optimizer-FP32StateAdamWeightDecay/Div-op5,Default/optimizer-FP32StateAdamWeightDecay/Div-op5,kernel_graph0,{18052},18052,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18052,18057,0x12c1c3819400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17906,607_Default/optimizer-FP32StateAdamWeightDecay/Mul-op29,Default/optimizer-FP32StateAdamWeightDecay/Mul-op29,kernel_graph0,{18055},18055,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18055,18063,0x12c1ae019000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17907,608_Default/optimizer-FP32StateAdamWeightDecay/Sub-op5,Default/optimizer-FP32StateAdamWeightDecay/Sub-op5,kernel_graph0,{18061},18061,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18064,18070,0x12c1a3418e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17911,612_Default/optimizer-FP32StateAdamWeightDecay/Mul-op30,Default/optimizer-FP32StateAdamWeightDecay/Mul-op30,kernel_graph0,{18068},18068,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18066,18074,0x12c1ae019000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17912,613_Default/optimizer-FP32StateAdamWeightDecay/Mul-op32,Default/optimizer-FP32StateAdamWeightDecay/Mul-op32,kernel_graph0,{18072},18072,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18068,18093,0x12c1b8c19200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17913,614_Default/learning_rate-CosineWithWarmUpLR/Add-op26,Default/learning_rate-CosineWithWarmUpLR/Add-op26,kernel_graph0,{18081-18091},18091,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18072,18091,0x12c1a3418e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17915,615_Default/learning_rate-CosineWithWarmUpLR/Add-op27,Default/learning_rate-CosineWithWarmUpLR/Add-op27,kernel_graph0,{18076-18089},18089,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18076,18080,0x12c1ae019000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17916,616_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op6,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op6,kernel_graph0,{18078},18078,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18078,18083,0x12c1c3819400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17917,617_Default/learning_rate-CosineWithWarmUpLR/Add-op28,Default/learning_rate-CosineWithWarmUpLR/Add-op28,kernel_graph0,{18081},18081,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18081,18086,0x12c1ae019000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17918,618_Default/optimizer-FP32StateAdamWeightDecay/Div-op6,Default/optimizer-FP32StateAdamWeightDecay/Div-op6,kernel_graph0,{18084},18084,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18084,18089,0x12c1c3819400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17918,619_Default/optimizer-FP32StateAdamWeightDecay/Mul-op34,Default/optimizer-FP32StateAdamWeightDecay/Mul-op34,kernel_graph0,{18087},18087,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18087,18095,0x12c1ae019000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17919,620_Default/optimizer-FP32StateAdamWeightDecay/Sub-op6,Default/optimizer-FP32StateAdamWeightDecay/Sub-op6,kernel_graph0,{18093},18093,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18096,18102,0x12c1a3418e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17923,624_Default/optimizer-FP32StateAdamWeightDecay/Mul-op35,Default/optimizer-FP32StateAdamWeightDecay/Mul-op35,kernel_graph0,{18100},18100,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18098,18106,0x12c1ae019000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17924,625_Default/optimizer-FP32StateAdamWeightDecay/Mul-op37,Default/optimizer-FP32StateAdamWeightDecay/Mul-op37,kernel_graph0,{18104},18104,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18100,18125,0x12c1b8c19200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17925,626_Default/learning_rate-CosineWithWarmUpLR/Add-op29,Default/learning_rate-CosineWithWarmUpLR/Add-op29,kernel_graph0,{18113-18123},18123,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18104,18123,0x12c1a3418e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17926,627_Default/learning_rate-CosineWithWarmUpLR/Add-op30,Default/learning_rate-CosineWithWarmUpLR/Add-op30,kernel_graph0,{18108-18121},18121,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18108,18112,0x12c1ae019000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17927,628_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op7,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op7,kernel_graph0,{18110},18110,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18110,18115,0x12c208813000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17928,629_Default/learning_rate-CosineWithWarmUpLR/Add-op31,Default/learning_rate-CosineWithWarmUpLR/Add-op31,kernel_graph0,{18113},18113,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18113,18118,0x12c1ae019000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17929,630_Default/optimizer-FP32StateAdamWeightDecay/Div-op7,Default/optimizer-FP32StateAdamWeightDecay/Div-op7,kernel_graph0,{18116},18116,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18116,18121,0x12c208813000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17930,631_Default/optimizer-FP32StateAdamWeightDecay/Mul-op39,Default/optimizer-FP32StateAdamWeightDecay/Mul-op39,kernel_graph0,{18119},18119,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18119,18127,0x12c1ae019000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17930,632_Default/optimizer-FP32StateAdamWeightDecay/Sub-op7,Default/optimizer-FP32StateAdamWeightDecay/Sub-op7,kernel_graph0,{18125},18125,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18128,18134,0x12c1e7936c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17932,636_Default/optimizer-FP32StateAdamWeightDecay/Mul-op40,Default/optimizer-FP32StateAdamWeightDecay/Mul-op40,kernel_graph0,{18132},18132,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18130,18138,0x12c1f1c12800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17932,637_Default/optimizer-FP32StateAdamWeightDecay/Mul-op42,Default/optimizer-FP32StateAdamWeightDecay/Mul-op42,kernel_graph0,{18136},18136,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18132,18157,0x12c1f5c12a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17932,638_Default/learning_rate-CosineWithWarmUpLR/Add-op32,Default/learning_rate-CosineWithWarmUpLR/Add-op32,kernel_graph0,{18145-18155},18155,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18136,18155,0x12c1e7936c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17933,639_Default/learning_rate-CosineWithWarmUpLR/Add-op33,Default/learning_rate-CosineWithWarmUpLR/Add-op33,kernel_graph0,{18140-18153},18153,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18140,18144,0x12c1f1c12800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17934,640_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op8,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op8,kernel_graph0,{18142},18142,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18142,18147,0x12c23a20fe00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17934,641_Default/learning_rate-CosineWithWarmUpLR/Add-op34,Default/learning_rate-CosineWithWarmUpLR/Add-op34,kernel_graph0,{18145},18145,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18145,18150,0x12c1f1c12800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17934,642_Default/optimizer-FP32StateAdamWeightDecay/Div-op8,Default/optimizer-FP32StateAdamWeightDecay/Div-op8,kernel_graph0,{18148},18148,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18148,18153,0x12c23a20fe00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17935,643_Default/optimizer-FP32StateAdamWeightDecay/Mul-op44,Default/optimizer-FP32StateAdamWeightDecay/Mul-op44,kernel_graph0,{18151},18151,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18151,18159,0x12c1f1c12800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17936,644_Default/optimizer-FP32StateAdamWeightDecay/Sub-op8,Default/optimizer-FP32StateAdamWeightDecay/Sub-op8,kernel_graph0,{18157},18157,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18160,18166,0x12c1e7936c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17939,648_Default/optimizer-FP32StateAdamWeightDecay/Mul-op45,Default/optimizer-FP32StateAdamWeightDecay/Mul-op45,kernel_graph0,{18164},18164,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18162,18170,0x12c23a20fe00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17939,649_Default/optimizer-FP32StateAdamWeightDecay/Mul-op47,Default/optimizer-FP32StateAdamWeightDecay/Mul-op47,kernel_graph0,{18168},18168,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18164,18189,0x12c1f1c12800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17940,650_Default/learning_rate-CosineWithWarmUpLR/Add-op35,Default/learning_rate-CosineWithWarmUpLR/Add-op35,kernel_graph0,{18177-18187},18187,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18168,18187,0x12c1f5c12a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17941,651_Default/learning_rate-CosineWithWarmUpLR/Add-op36,Default/learning_rate-CosineWithWarmUpLR/Add-op36,kernel_graph0,{18172-18185},18185,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18172,18176,0x12c1e7936c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17943,652_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op9,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op9,kernel_graph0,{18174},18174,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18174,18179,0x12c23a20fe00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17944,653_Default/learning_rate-CosineWithWarmUpLR/Add-op37,Default/learning_rate-CosineWithWarmUpLR/Add-op37,kernel_graph0,{18177},18177,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18177,18182,0x12c1e7936c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17945,654_Default/optimizer-FP32StateAdamWeightDecay/Div-op9,Default/optimizer-FP32StateAdamWeightDecay/Div-op9,kernel_graph0,{18180},18180,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18180,18185,0x12c23a20fe00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17947,655_Default/optimizer-FP32StateAdamWeightDecay/Mul-op49,Default/optimizer-FP32StateAdamWeightDecay/Mul-op49,kernel_graph0,{18183},18183,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18183,18191,0x12c1e7936c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17948,656_Default/optimizer-FP32StateAdamWeightDecay/Sub-op9,Default/optimizer-FP32StateAdamWeightDecay/Sub-op9,kernel_graph0,{18189},18189,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18192,18198,0x12c1e7936c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17951,660_Default/optimizer-FP32StateAdamWeightDecay/Mul-op50,Default/optimizer-FP32StateAdamWeightDecay/Mul-op50,kernel_graph0,{18196},18196,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18194,18202,0x12c23a20fe00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17953,661_Default/optimizer-FP32StateAdamWeightDecay/Mul-op52,Default/optimizer-FP32StateAdamWeightDecay/Mul-op52,kernel_graph0,{18200},18200,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18196,18221,0x12c1f1c12800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17954,662_Default/learning_rate-CosineWithWarmUpLR/Add-op38,Default/learning_rate-CosineWithWarmUpLR/Add-op38,kernel_graph0,{18209-18219},18219,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18200,18219,0x12c1f5c12a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17954,663_Default/learning_rate-CosineWithWarmUpLR/Add-op39,Default/learning_rate-CosineWithWarmUpLR/Add-op39,kernel_graph0,{18204-18217},18217,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18204,18208,0x12c23a20fe00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17956,664_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op10,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op10,kernel_graph0,{18206},18206,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18206,18211,0x12c1e3936a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17957,665_Default/learning_rate-CosineWithWarmUpLR/Add-op40,Default/learning_rate-CosineWithWarmUpLR/Add-op40,kernel_graph0,{18209},18209,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18209,18214,0x12c23a20fe00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17957,666_Default/optimizer-FP32StateAdamWeightDecay/Div-op10,Default/optimizer-FP32StateAdamWeightDecay/Div-op10,kernel_graph0,{18212},18212,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18212,18217,0x12c1e3936a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17959,667_Default/optimizer-FP32StateAdamWeightDecay/Mul-op54,Default/optimizer-FP32StateAdamWeightDecay/Mul-op54,kernel_graph0,{18215},18215,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18215,18223,0x12c23a20fe00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17960,668_Default/optimizer-FP32StateAdamWeightDecay/Sub-op10,Default/optimizer-FP32StateAdamWeightDecay/Sub-op10,kernel_graph0,{18221},18221,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18224,18230,0x12c23a20fe00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17962,672_Default/optimizer-FP32StateAdamWeightDecay/Mul-op55,Default/optimizer-FP32StateAdamWeightDecay/Mul-op55,kernel_graph0,{18228},18228,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18226,18234,0x12c1f1c12800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17962,673_Default/optimizer-FP32StateAdamWeightDecay/Mul-op57,Default/optimizer-FP32StateAdamWeightDecay/Mul-op57,kernel_graph0,{18232},18232,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18228,18253,0x12c1f5c12a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17962,674_Default/learning_rate-CosineWithWarmUpLR/Add-op41,Default/learning_rate-CosineWithWarmUpLR/Add-op41,kernel_graph0,{18241-18251},18251,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18232,18251,0x12c23a20fe00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17963,675_Default/learning_rate-CosineWithWarmUpLR/Add-op42,Default/learning_rate-CosineWithWarmUpLR/Add-op42,kernel_graph0,{18236-18249},18249,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18236,18240,0x12c1f1c12800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17964,676_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op11,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op11,kernel_graph0,{18238},18238,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18238,18243,0x12c204812e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17964,677_Default/learning_rate-CosineWithWarmUpLR/Add-op43,Default/learning_rate-CosineWithWarmUpLR/Add-op43,kernel_graph0,{18241},18241,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18241,18246,0x12c1f1c12800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17964,678_Default/optimizer-FP32StateAdamWeightDecay/Div-op11,Default/optimizer-FP32StateAdamWeightDecay/Div-op11,kernel_graph0,{18244},18244,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18244,18249,0x12c204812e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17965,679_Default/optimizer-FP32StateAdamWeightDecay/Mul-op59,Default/optimizer-FP32StateAdamWeightDecay/Mul-op59,kernel_graph0,{18247},18247,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18247,18255,0x12c1f1c12800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17966,680_Default/optimizer-FP32StateAdamWeightDecay/Sub-op11,Default/optimizer-FP32StateAdamWeightDecay/Sub-op11,kernel_graph0,{18253},18253,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18256,18262,0x12c204812e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17968,684_Default/optimizer-FP32StateAdamWeightDecay/Mul-op60,Default/optimizer-FP32StateAdamWeightDecay/Mul-op60,kernel_graph0,{18260},18260,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18258,18266,0x12c19f418c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17969,685_Default/optimizer-FP32StateAdamWeightDecay/Mul-op62,Default/optimizer-FP32StateAdamWeightDecay/Mul-op62,kernel_graph0,{18264},18264,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18260,18285,0x12c1aa018e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17969,686_Default/learning_rate-CosineWithWarmUpLR/Add-op44,Default/learning_rate-CosineWithWarmUpLR/Add-op44,kernel_graph0,{18273-18283},18283,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18264,18283,0x12c204812e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17970,687_Default/learning_rate-CosineWithWarmUpLR/Add-op45,Default/learning_rate-CosineWithWarmUpLR/Add-op45,kernel_graph0,{18268-18281},18281,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18268,18272,0x12c19f418c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17971,688_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op12,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op12,kernel_graph0,{18270},18270,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18270,18275,0x12c22361ba00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17971,689_Default/learning_rate-CosineWithWarmUpLR/Add-op46,Default/learning_rate-CosineWithWarmUpLR/Add-op46,kernel_graph0,{18273},18273,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18273,18278,0x12c19f418c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17972,690_Default/optimizer-FP32StateAdamWeightDecay/Div-op12,Default/optimizer-FP32StateAdamWeightDecay/Div-op12,kernel_graph0,{18276},18276,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18276,18281,0x12c22361ba00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17973,691_Default/optimizer-FP32StateAdamWeightDecay/Mul-op64,Default/optimizer-FP32StateAdamWeightDecay/Mul-op64,kernel_graph0,{18279},18279,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18279,18287,0x12c19f418c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17975,692_Default/optimizer-FP32StateAdamWeightDecay/Sub-op12,Default/optimizer-FP32StateAdamWeightDecay/Sub-op12,kernel_graph0,{18285},18285,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18288,18294,0x12c22361ba00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17980,696_Default/optimizer-FP32StateAdamWeightDecay/Mul-op65,Default/optimizer-FP32StateAdamWeightDecay/Mul-op65,kernel_graph0,{18292},18292,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18290,18298,0x12c204812e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17981,697_Default/optimizer-FP32StateAdamWeightDecay/Mul-op67,Default/optimizer-FP32StateAdamWeightDecay/Mul-op67,kernel_graph0,{18296},18296,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18292,18317,0x12c19f418c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17982,698_Default/learning_rate-CosineWithWarmUpLR/Add-op47,Default/learning_rate-CosineWithWarmUpLR/Add-op47,kernel_graph0,{18305-18315},18315,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18296,18315,0x12c22361ba00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17982,699_Default/learning_rate-CosineWithWarmUpLR/Add-op48,Default/learning_rate-CosineWithWarmUpLR/Add-op48,kernel_graph0,{18300-18313},18313,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18300,18304,0x12c22e21fe00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17983,700_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op13,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op13,kernel_graph0,{18302},18302,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18302,18307,0x12c204812e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17983,701_Default/learning_rate-CosineWithWarmUpLR/Add-op49,Default/learning_rate-CosineWithWarmUpLR/Add-op49,kernel_graph0,{18305},18305,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18305,18310,0x12c22e21fe00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17985,702_Default/optimizer-FP32StateAdamWeightDecay/Div-op13,Default/optimizer-FP32StateAdamWeightDecay/Div-op13,kernel_graph0,{18308},18308,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18308,18313,0x12c204812e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17986,703_Default/optimizer-FP32StateAdamWeightDecay/Mul-op69,Default/optimizer-FP32StateAdamWeightDecay/Mul-op69,kernel_graph0,{18311},18311,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18311,18319,0x12c22e21fe00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17986,704_Default/optimizer-FP32StateAdamWeightDecay/Sub-op13,Default/optimizer-FP32StateAdamWeightDecay/Sub-op13,kernel_graph0,{18317},18317,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18320,18326,0x12c22361ba00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17991,708_Default/optimizer-FP32StateAdamWeightDecay/Mul-op70,Default/optimizer-FP32StateAdamWeightDecay/Mul-op70,kernel_graph0,{18324},18324,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18322,18330,0x12c22e21fe00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17992,709_Default/optimizer-FP32StateAdamWeightDecay/Mul-op72,Default/optimizer-FP32StateAdamWeightDecay/Mul-op72,kernel_graph0,{18328},18328,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18324,18349,0x12c204812e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17992,710_Default/learning_rate-CosineWithWarmUpLR/Add-op50,Default/learning_rate-CosineWithWarmUpLR/Add-op50,kernel_graph0,{18337-18347},18347,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18328,18347,0x12c22361ba00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17993,711_Default/learning_rate-CosineWithWarmUpLR/Add-op51,Default/learning_rate-CosineWithWarmUpLR/Add-op51,kernel_graph0,{18332-18345},18345,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18332,18336,0x12c1f1c12800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17994,712_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op14,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op14,kernel_graph0,{18334},18334,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18334,18339,0x12c22e21fe00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17994,713_Default/learning_rate-CosineWithWarmUpLR/Add-op52,Default/learning_rate-CosineWithWarmUpLR/Add-op52,kernel_graph0,{18337},18337,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18337,18342,0x12c1f1c12800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17994,714_Default/optimizer-FP32StateAdamWeightDecay/Div-op14,Default/optimizer-FP32StateAdamWeightDecay/Div-op14,kernel_graph0,{18340},18340,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18340,18345,0x12c22e21fe00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17995,715_Default/optimizer-FP32StateAdamWeightDecay/Mul-op74,Default/optimizer-FP32StateAdamWeightDecay/Mul-op74,kernel_graph0,{18343},18343,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18343,18351,0x12c1f1c12800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17995,716_Default/optimizer-FP32StateAdamWeightDecay/Sub-op14,Default/optimizer-FP32StateAdamWeightDecay/Sub-op14,kernel_graph0,{18349},18349,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18352,18358,0x12c1f1c12800,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17997,720_Default/optimizer-FP32StateAdamWeightDecay/Mul-op75,Default/optimizer-FP32StateAdamWeightDecay/Mul-op75,kernel_graph0,{18356},18356,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18354,18362,0x12c19f418c00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17998,721_Default/optimizer-FP32StateAdamWeightDecay/Mul-op77,Default/optimizer-FP32StateAdamWeightDecay/Mul-op77,kernel_graph0,{18360},18360,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18356,18381,0x12c1be818e00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17998,722_Default/learning_rate-CosineWithWarmUpLR/Add-op53,Default/learning_rate-CosineWithWarmUpLR/Add-op53,kernel_graph0,{18369-18379},18379,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18360,18379,0x12c1f1c12800,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17998,723_Default/learning_rate-CosineWithWarmUpLR/Add-op54,Default/learning_rate-CosineWithWarmUpLR/Add-op54,kernel_graph0,{18364-18377},18377,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18364,18368,0x12c180018a00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,17999,724_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op15,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op15,kernel_graph0,{18366},18366,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18366,18371,0x12c19f418c00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18000,725_Default/learning_rate-CosineWithWarmUpLR/Add-op55,Default/learning_rate-CosineWithWarmUpLR/Add-op55,kernel_graph0,{18369},18369,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18369,18374,0x12c180018a00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18001,726_Default/optimizer-FP32StateAdamWeightDecay/Div-op15,Default/optimizer-FP32StateAdamWeightDecay/Div-op15,kernel_graph0,{18372},18372,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18372,18377,0x12c19f418c00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18001,727_Default/optimizer-FP32StateAdamWeightDecay/Mul-op79,Default/optimizer-FP32StateAdamWeightDecay/Mul-op79,kernel_graph0,{18375},18375,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18375,18383,0x12c180018a00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18002,728_Default/optimizer-FP32StateAdamWeightDecay/Sub-op15,Default/optimizer-FP32StateAdamWeightDecay/Sub-op15,kernel_graph0,{18381},18381,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18384,18390,0x12c22361ba00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18004,732_Default/optimizer-FP32StateAdamWeightDecay/Mul-op80,Default/optimizer-FP32StateAdamWeightDecay/Mul-op80,kernel_graph0,{18388},18388,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18386,18394,0x12c22361fc00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18005,733_Default/optimizer-FP32StateAdamWeightDecay/Mul-op82,Default/optimizer-FP32StateAdamWeightDecay/Mul-op82,kernel_graph0,{18392},18392,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18388,18413,0x12c223623e00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18007,734_Default/learning_rate-CosineWithWarmUpLR/Add-op56,Default/learning_rate-CosineWithWarmUpLR/Add-op56,kernel_graph0,{18401-18411},18411,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18392,18411,0x12c223617800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18008,735_Default/learning_rate-CosineWithWarmUpLR/Add-op57,Default/learning_rate-CosineWithWarmUpLR/Add-op57,kernel_graph0,{18396-18409},18409,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18396,18400,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18009,736_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op16,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op16,kernel_graph0,{18398},18398,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18398,18403,0x12c22361ba00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18011,737_Default/learning_rate-CosineWithWarmUpLR/Add-op58,Default/learning_rate-CosineWithWarmUpLR/Add-op58,kernel_graph0,{18401},18401,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18401,18406,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18012,738_Default/optimizer-FP32StateAdamWeightDecay/Div-op16,Default/optimizer-FP32StateAdamWeightDecay/Div-op16,kernel_graph0,{18404},18404,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18404,18409,0x12c22361ba00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18013,739_Default/optimizer-FP32StateAdamWeightDecay/Mul-op84,Default/optimizer-FP32StateAdamWeightDecay/Mul-op84,kernel_graph0,{18407},18407,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18407,18415,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18013,740_Default/optimizer-FP32StateAdamWeightDecay/Sub-op16,Default/optimizer-FP32StateAdamWeightDecay/Sub-op16,kernel_graph0,{18413},18413,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18416,18422,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18017,744_Default/optimizer-FP32StateAdamWeightDecay/Mul-op85,Default/optimizer-FP32StateAdamWeightDecay/Mul-op85,kernel_graph0,{18420},18420,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18418,18426,0x12c223617800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18018,745_Default/optimizer-FP32StateAdamWeightDecay/Mul-op87,Default/optimizer-FP32StateAdamWeightDecay/Mul-op87,kernel_graph0,{18424},18424,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18420,18445,0x12c22361ba00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18019,746_Default/learning_rate-CosineWithWarmUpLR/Add-op59,Default/learning_rate-CosineWithWarmUpLR/Add-op59,kernel_graph0,{18433-18443},18443,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18424,18443,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18020,747_Default/learning_rate-CosineWithWarmUpLR/Add-op60,Default/learning_rate-CosineWithWarmUpLR/Add-op60,kernel_graph0,{18428-18441},18441,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18428,18432,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18021,748_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op17,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op17,kernel_graph0,{18430},18430,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18430,18435,0x12c223613600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18023,749_Default/learning_rate-CosineWithWarmUpLR/Add-op61,Default/learning_rate-CosineWithWarmUpLR/Add-op61,kernel_graph0,{18433},18433,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18433,18438,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18023,750_Default/optimizer-FP32StateAdamWeightDecay/Div-op17,Default/optimizer-FP32StateAdamWeightDecay/Div-op17,kernel_graph0,{18436},18436,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18436,18441,0x12c223613600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18024,751_Default/optimizer-FP32StateAdamWeightDecay/Mul-op89,Default/optimizer-FP32StateAdamWeightDecay/Mul-op89,kernel_graph0,{18439},18439,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18439,18447,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18025,752_Default/optimizer-FP32StateAdamWeightDecay/Sub-op17,Default/optimizer-FP32StateAdamWeightDecay/Sub-op17,kernel_graph0,{18445},18445,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18448,18454,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18026,756_Default/optimizer-FP32StateAdamWeightDecay/Mul-op90,Default/optimizer-FP32StateAdamWeightDecay/Mul-op90,kernel_graph0,{18452},18452,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18450,18458,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18027,757_Default/optimizer-FP32StateAdamWeightDecay/Mul-op92,Default/optimizer-FP32StateAdamWeightDecay/Mul-op92,kernel_graph0,{18456},18456,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18452,18477,0x12c223613600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18028,758_Default/learning_rate-CosineWithWarmUpLR/Add-op62,Default/learning_rate-CosineWithWarmUpLR/Add-op62,kernel_graph0,{18465-18475},18475,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18456,18475,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18028,759_Default/learning_rate-CosineWithWarmUpLR/Add-op63,Default/learning_rate-CosineWithWarmUpLR/Add-op63,kernel_graph0,{18460-18473},18473,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18460,18464,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18028,760_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op18,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op18,kernel_graph0,{18462},18462,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18462,18467,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18029,761_Default/learning_rate-CosineWithWarmUpLR/Add-op64,Default/learning_rate-CosineWithWarmUpLR/Add-op64,kernel_graph0,{18465},18465,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18465,18470,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18029,762_Default/optimizer-FP32StateAdamWeightDecay/Div-op18,Default/optimizer-FP32StateAdamWeightDecay/Div-op18,kernel_graph0,{18468},18468,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18468,18473,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18030,763_Default/optimizer-FP32StateAdamWeightDecay/Mul-op94,Default/optimizer-FP32StateAdamWeightDecay/Mul-op94,kernel_graph0,{18471},18471,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18471,18479,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18030,764_Default/optimizer-FP32StateAdamWeightDecay/Sub-op18,Default/optimizer-FP32StateAdamWeightDecay/Sub-op18,kernel_graph0,{18477},18477,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18480,18486,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18033,768_Default/optimizer-FP32StateAdamWeightDecay/Mul-op97,Default/optimizer-FP32StateAdamWeightDecay/Mul-op97,kernel_graph0,{18484},18484,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18482,18490,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18034,769_Default/optimizer-FP32StateAdamWeightDecay/Mul-op95,Default/optimizer-FP32StateAdamWeightDecay/Mul-op95,kernel_graph0,{18488},18488,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18484,18507,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18035,770_Default/learning_rate-CosineWithWarmUpLR/Add-op66,Default/learning_rate-CosineWithWarmUpLR/Add-op66,kernel_graph0,{18492-18505},18505,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18488,18509,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18035,771_Default/learning_rate-CosineWithWarmUpLR/Add-op65,Default/learning_rate-CosineWithWarmUpLR/Add-op65,kernel_graph0,{18497-18507},18507,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18492,18496,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18036,772_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op19,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op19,kernel_graph0,{18494},18494,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18494,18499,0x12c180014800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18037,773_Default/learning_rate-CosineWithWarmUpLR/Add-op67,Default/learning_rate-CosineWithWarmUpLR/Add-op67,kernel_graph0,{18497},18497,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18497,18502,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18037,774_Default/optimizer-FP32StateAdamWeightDecay/Div-op19,Default/optimizer-FP32StateAdamWeightDecay/Div-op19,kernel_graph0,{18500},18500,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18500,18505,0x12c180014800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18040,775_Default/optimizer-FP32StateAdamWeightDecay/Mul-op99,Default/optimizer-FP32StateAdamWeightDecay/Mul-op99,kernel_graph0,{18503},18503,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18503,18511,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18041,776_Default/optimizer-FP32StateAdamWeightDecay/Sub-op19,Default/optimizer-FP32StateAdamWeightDecay/Sub-op19,kernel_graph0,{18509},18509,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18512,18518,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18045,780_Default/optimizer-FP32StateAdamWeightDecay/Mul-op102,Default/optimizer-FP32StateAdamWeightDecay/Mul-op102,kernel_graph0,{18516},18516,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18514,18522,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18046,781_Default/optimizer-FP32StateAdamWeightDecay/Mul-op100,Default/optimizer-FP32StateAdamWeightDecay/Mul-op100,kernel_graph0,{18520},18520,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18516,18544,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18047,782_Default/learning_rate-CosineWithWarmUpLR/Add-op69,Default/learning_rate-CosineWithWarmUpLR/Add-op69,kernel_graph0,{18524-18542},18542,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18520,18542,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18047,783_Default/learning_rate-CosineWithWarmUpLR/Add-op68,Default/learning_rate-CosineWithWarmUpLR/Add-op68,kernel_graph0,{18529-18540},18540,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18524,18528,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18049,784_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op20,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op20,kernel_graph0,{18526},18526,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18526,18531,0x12c180010600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18050,785_Default/learning_rate-CosineWithWarmUpLR/Add-op70,Default/learning_rate-CosineWithWarmUpLR/Add-op70,kernel_graph0,{18529},18529,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18529,18535,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18050,786_Default/optimizer-FP32StateAdamWeightDecay/Div-op20,Default/optimizer-FP32StateAdamWeightDecay/Div-op20,kernel_graph0,{18532},18532,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18532,18538,0x12c180010600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18052,787_Default/optimizer-FP32StateAdamWeightDecay/Mul-op104,Default/optimizer-FP32StateAdamWeightDecay/Mul-op104,kernel_graph0,{18536},18536,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18536,18540,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18053,788_Default/optimizer-FP32StateAdamWeightDecay/Sub-op20,Default/optimizer-FP32StateAdamWeightDecay/Sub-op20,kernel_graph0,{18538},18538,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18545,20844,0x12c180008c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18056,792_Default/TensorMove-op2,Default/TensorMove-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18549,20842,0x12c75d35ee00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/output_actor.cc,689,Other,18547,kernel_graph_0_OutputActor,UpdateOutputDeviceAddress,,{},,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 18555,18959,0x12c75d35f000,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Kernel,18554,0_Default/network-LlamaForCausalLM/StridedSlice-op0,inputs0,0_Default/network-LlamaForCausalLM/StridedSlice-op0,{18556-18957},18957,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,1,0, 18556,19538,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18554,0_Default/network-LlamaForCausalLM/StridedSlice-op0,inputs0,0_Default/network-LlamaForCausalLM/StridedSlice-op0,{18558-18560-19535},19535,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18558,18564,0x12c18000c000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18556,1_Default/network-LlamaForCausalLM/model-LlamaModel/tok_embeddings-LlamaEmbedding/Gather-op0,Default/network-LlamaForCausalLM/model-LlamaModel/tok_embeddings-LlamaEmbedding/Gather-op0,kernel_graph0,{18562},18562,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18560,18570,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,8704,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18557,2_Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/NotEqual-op0,Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/NotEqual-op0,kernel_graph0,{18565-18568},18568,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18562,19516,0x12c18800c200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18557,3_Default/network-LlamaForCausalLM/model-LlamaModel/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/Cast-op0,kernel_graph0,{18574-18703-19514},19514,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18565,18993,0x12c18000c000,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18557,4_Default/network-LlamaForCausalLM/Cast-op1,Default/network-LlamaForCausalLM/Cast-op1,kernel_graph0,{18566-18585-18946-18990},18990,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18568,18599,0x12c180014200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18557,6_Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Cast-op0,kernel_graph0,{18593-18597},18597,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18571,19442,0x12c180018400,0,DefaultEnhancedAscendMemoryPool,524800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18557,7_Default/network-LlamaForCausalLM/model-LlamaModel/freqs_mgr-FreqsMgr/StridedSlice-op0,Default/network-LlamaForCausalLM/model-LlamaModel/freqs_mgr-FreqsMgr/StridedSlice-op0,kernel_graph0,{18572-18636-18659-18789-18804-19177-19198-19419-19440},19440,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18574,18580,0x12c18c00c400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18557,9_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op0,kernel_graph0,{18578},18578,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18576,18584,0x12c19400c600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18558,10_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{18582},18582,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18577,19523,0x12c180098600,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18558,10_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{19519},19519,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18578,18581,0x12c1800a0800,0,DefaultEnhancedAscendMemoryPool,16777728,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18558,10_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18582,19494,0x12c1800a0800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18559,11_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op1,kernel_graph0,{18607-18612-18627-18650-19464-19478-19492},19492,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18585,18593,0x12c180008800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18559,12_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/ReduceSum-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/ReduceSum-op0,kernel_graph0,{18591},18591,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18587,18589,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,18587,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18588,18590,0x12c180009200,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,18588,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18591,19004,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18559,13_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Add-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Add-op0,kernel_graph0,{18934-19001},19001,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18595,18600,0x12c1840a0a00,0,DefaultEnhancedAscendMemoryPool,2097664,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18560,15_Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/ExpandDims-op0,Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/ExpandDims-op0,kernel_graph0,{18597},18597,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18597,18603,0x12c1842a0c00,0,DefaultEnhancedAscendMemoryPool,16777728,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18561,16_Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Mul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Mul-op0,kernel_graph0,{18601},18601,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18601,18607,0x12c1852a0e00,0,DefaultEnhancedAscendMemoryPool,16777728,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18561,17_Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Sub-op0,Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Sub-op0,kernel_graph0,{18603-18605},18605,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18605,19400,0x12c1840a0a00,0,DefaultEnhancedAscendMemoryPool,8389120,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18562,19_Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Cast-op1,kernel_graph0,{18688-18835-19153-19394},19394,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18609,19488,0x12c1848a0c00,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18563,21_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/Cast-op0,kernel_graph0,{18612-19485},19485,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18611,18619,0x12c18c00c400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18565,22_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,kernel_graph0,{18614-18617},18617,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18612,18614,0x12c1868a0e00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18565,22_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18616,18623,0x12c19000c600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18566,24_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,kernel_graph0,{18621},18621,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18617,18620,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18566,24_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18621,18640,0x12c19400c800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18566,25_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op2,kernel_graph0,{18636-18638},18638,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18624,19503,0x12c18c00c400,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18567,26_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/Cast-op0,kernel_graph0,{18627-19500},19500,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18626,18634,0x12c18e00c600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18568,27_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,kernel_graph0,{18629-18632},18632,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18627,18629,0x12c1868a0e00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18568,27_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18631,18643,0x12c19c00ca00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18570,29_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,kernel_graph0,{18641},18641,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18632,18635,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18570,29_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18636,18679,0x12c1a000cc00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18571,30_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op2,kernel_graph0,{18677},18677,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18638,18676,0x12c1a800ce00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18572,31_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op1,kernel_graph0,{18674},18674,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18641,18663,0x12c18e00c600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18572,32_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op0,kernel_graph0,{18659-18661},18661,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18644,19445,0x12c1868a0e00,0,DefaultEnhancedAscendMemoryPool,524800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18572,33_Default/network-LlamaForCausalLM/model-LlamaModel/freqs_mgr-FreqsMgr/StridedSlice-op1,Default/network-LlamaForCausalLM/model-LlamaModel/freqs_mgr-FreqsMgr/StridedSlice-op1,kernel_graph0,{18645-18664-18674-18794-18809-19179-19200-19421-19443},19443,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18647,19474,0x12c19600c800,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18572,35_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/Cast-op0,kernel_graph0,{18650-19471},19471,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18649,18657,0x12c19800ca00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18573,36_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,kernel_graph0,{18652-18655},18655,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18650,18652,0x12c186921000,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18573,36_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18654,19398,0x12c1b000d000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18575,38_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,kernel_graph0,{18688-19394},19394,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18655,18658,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18575,38_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18659,18669,0x12c19800ca00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18575,39_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op0,kernel_graph0,{18667},18667,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18661,18666,0x12c1b400d200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18578,40_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op0,kernel_graph0,{18664},18664,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18664,18670,0x12c18e00c600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18579,41_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op1,kernel_graph0,{18667},18667,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18667,18673,0x12c1b400d200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18579,42_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op0,kernel_graph0,{18671},18671,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18671,19396,0x12c18e00c600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18582,43_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op1,kernel_graph0,{18688-19394},19394,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18674,18680,0x12c19800ca00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18583,44_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op3,kernel_graph0,{18677},18677,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18677,18683,0x12c1a800ce00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18583,45_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op1,kernel_graph0,{18681},18681,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18681,19397,0x12c19800ca00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18585,46_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op3,kernel_graph0,{18688-19394},19394,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18684,19401,0x12c186921000,0,DefaultEnhancedAscendMemoryPool,8389120,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18586,47_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{19394},19394,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18685,19402,0x12c187121200,0,DefaultEnhancedAscendMemoryPool,8389120,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18586,47_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{19394},19394,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18686,19403,0x12c180008800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18586,47_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{19394},19394,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18687,19404,0x12c19c00cc00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18586,47_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{18692-19394},19394,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18688,18690,0x12c1a000ce00,0,DefaultEnhancedAscendMemoryPool,84936192,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18586,47_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18691,19374,0x12c1a000ce00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18586,48_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,kernel_graph0,{18694-18699-19372},19372,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18692,18694,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18586,48_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18696,19382,0x12c19200c800,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18587,50_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/Cast-op0,kernel_graph0,{18699-19380},19380,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18698,18705,0x12c1a400d000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18587,51_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,kernel_graph0,{18701-18703},18703,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18699,18701,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18587,51_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18703,19354,0x12c1a800d200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18588,53_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/Add-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/Add-op0,kernel_graph0,{18706-18744-19352},19352,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18706,18712,0x12c1b400d200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18590,54_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op0,kernel_graph0,{18710},18710,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18708,18716,0x12c1bc00d400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18591,55_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{18714},18714,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18709,19361,0x12c187921400,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18591,55_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{19357},19357,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18710,18713,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,16777728,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18591,55_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18714,19333,0x12c1a400d000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18592,56_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op1,kernel_graph0,{18716-18721-18728-19317-19331},19331,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18718,19327,0x12c1b400d200,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18593,58_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/Cast-op0,kernel_graph0,{18721-19324},19324,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18720,19309,0x12c1b960d400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18594,59_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,kernel_graph0,{18723-18734-19307},19307,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18721,18723,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18594,59_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18725,19342,0x12c1c420d600,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18596,61_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/Cast-op0,kernel_graph0,{18728-19339},19339,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18727,19315,0x12c1c980d800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18596,62_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,kernel_graph0,{18731-19312},19312,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18728,18730,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18596,62_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18731,19305,0x12c1d440da00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18597,63_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/SiLU-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/SiLU-op0,kernel_graph0,{18732-18734-19303},19303,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18734,19292,0x12c1df00dc00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18600,65_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/Mul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/Mul-op0,kernel_graph0,{18735-18740-19290},19290,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18737,19300,0x12c1e9c0de00,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18602,67_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/Cast-op0,kernel_graph0,{18740-19298},19298,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18739,18746,0x12c1ef20e000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18603,68_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,kernel_graph0,{18742-18744},18744,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18740,18742,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18603,68_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18744,19272,0x12c1f320e200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18605,70_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/Add-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/Add-op1,kernel_graph0,{18747-18850-19270},19270,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18747,18753,0x12c1f720e400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18606,71_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op0,kernel_graph0,{18751},18751,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18749,18757,0x12c1ff20e600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18606,72_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{18755},18755,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18750,19279,0x12c187929600,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18606,72_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{19275},19275,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18751,18754,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,16777728,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18606,72_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18755,19250,0x12c1ef20e000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18607,73_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op1,kernel_graph0,{18757-18762-18777-18822-19220-19234-19248},19248,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18759,19244,0x12c1ac00d400,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18609,75_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/Cast-op0,kernel_graph0,{18762-19241},19241,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18761,18769,0x12c1f720e400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18610,76_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,kernel_graph0,{18764-18767},18767,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18762,18764,0x12c1ae00d600,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18610,76_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18766,18773,0x12c1fb20e600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18612,78_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,kernel_graph0,{18771},18771,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18767,18770,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18612,78_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18771,18793,0x12c1ff20e800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18612,79_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op2,kernel_graph0,{18789-18791},18791,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18774,19259,0x12c1f720e400,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18613,80_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/Cast-op0,kernel_graph0,{18777-19256},19256,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18776,18784,0x12c1f920e600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18614,81_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,kernel_graph0,{18779-18782},18782,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18777,18779,0x12c1ae00d600,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18614,81_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18781,18788,0x12c20720ea00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18615,83_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,kernel_graph0,{18786},18786,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18782,18785,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18615,83_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18786,18808,0x12c20b20ec00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18617,84_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op0,kernel_graph0,{18804-18806},18806,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18789,18799,0x12c21320ee00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18618,85_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op2,kernel_graph0,{18797},18797,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18791,18796,0x12c21b20f000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18618,86_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op1,kernel_graph0,{18794},18794,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18794,18800,0x12c1f920e600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18620,87_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op3,kernel_graph0,{18797},18797,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18797,18803,0x12c20120e800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18621,88_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op1,kernel_graph0,{18801},18801,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18801,19156,0x12c1f920e600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18622,89_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op3,kernel_graph0,{18835-19153},19153,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18804,18814,0x12c1fd20e800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18623,90_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op0,kernel_graph0,{18812},18812,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18806,18811,0x12c21320ee00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18624,91_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op0,kernel_graph0,{18809},18809,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18809,18815,0x12c20520ea00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18625,92_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op1,kernel_graph0,{18812},18812,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18812,18818,0x12c20d20ec00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18625,93_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op0,kernel_graph0,{18816},18816,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18816,19155,0x12c1fd20e800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18627,94_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op1,kernel_graph0,{18835-19153},19153,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18819,19230,0x12c20120ea00,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18628,95_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/Cast-op0,kernel_graph0,{18822-19227},19227,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18821,18829,0x12c20320ec00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18628,96_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,kernel_graph0,{18824-18827},18827,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18822,18824,0x12c1ae00d600,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18628,96_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18826,19157,0x12c20720ee00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18630,98_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,kernel_graph0,{18835-19153},19153,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18827,18830,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18630,98_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18831,19159,0x12c1ae00d600,0,DefaultEnhancedAscendMemoryPool,8389120,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18632,99_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{19153},19153,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18832,19160,0x12c1ae80d800,0,DefaultEnhancedAscendMemoryPool,8389120,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18632,99_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{19153},19153,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18833,19161,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18632,99_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{19153},19153,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18834,19162,0x12c20320ec00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18632,99_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{18839-19153},19153,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18835,18837,0x12c20b20f000,0,DefaultEnhancedAscendMemoryPool,84936192,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18632,99_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18838,19133,0x12c20b20f000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18633,100_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,kernel_graph0,{18841-18846-19131},19131,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18839,18841,0x12c180009200,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18633,100_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18843,19141,0x12c20f20f200,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18636,102_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/Cast-op0,kernel_graph0,{18846-19139},19139,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18845,18852,0x12c21120f400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18637,103_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,kernel_graph0,{18848-18850},18850,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18846,18848,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18637,103_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18850,19108,0x12c21520f600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18638,105_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/Add-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/Add-op0,kernel_graph0,{18853-18891-19106},19106,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18853,18859,0x12c21920f800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18639,106_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op0,kernel_graph0,{18857},18857,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18855,18863,0x12c22120fa00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18639,107_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{18861},18861,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18856,19115,0x12c187931800,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18639,107_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{19111},19111,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18857,18860,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,16777728,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18639,107_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18861,19087,0x12c21120f400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18641,108_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op1,kernel_graph0,{18863-18868-18875-19071-19085},19085,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18865,19081,0x12c21920f800,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18642,110_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/Cast-op0,kernel_graph0,{18868-19078},19078,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18867,19063,0x12c21e80fa00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18644,111_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,kernel_graph0,{18870-18881-19061},19061,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18868,18870,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18644,111_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18872,19096,0x12c22940fc00,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18645,113_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/Cast-op0,kernel_graph0,{18875-19093},19093,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18874,19069,0x12c22ea0fe00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18645,114_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,kernel_graph0,{18878-19066},19066,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18875,18877,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18645,114_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18878,19059,0x12c239610000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18645,115_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/SiLU-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/SiLU-op0,kernel_graph0,{18879-18881-19057},19057,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18881,19046,0x12c244210200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18646,117_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/Mul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/Mul-op0,kernel_graph0,{18882-18887-19044},19044,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18884,19054,0x12c24ee10400,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18648,119_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/Cast-op0,kernel_graph0,{18887-19052},19052,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18886,18893,0x12c254410600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18648,120_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,kernel_graph0,{18889-18891},18891,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18887,18889,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18648,120_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18891,19025,0x12c258410800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18651,122_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/Add-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/Add-op1,kernel_graph0,{18894-19023},19023,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18894,18900,0x12c25c410a00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18651,123_Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Cast-op0,kernel_graph0,{18898},18898,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18896,18904,0x12c264410c00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18652,124_Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{18902},18902,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18897,19032,0x12c187939a00,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18652,124_Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{19028},19028,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18898,18901,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,16777728,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18652,124_Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18902,19008,0x12c254410600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18653,125_Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Cast-op1,kernel_graph0,{18904-18909-19006},19006,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18906,19017,0x12c25c410a00,0,DefaultEnhancedAscendMemoryPool,262144512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18656,127_Default/network-LlamaForCausalLM/lm_head-Linear/Cast-op0,Default/network-LlamaForCausalLM/lm_head-Linear/Cast-op0,kernel_graph0,{18909-19014},19014,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18908,18914,0x12c26be10c00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18658,128_Default/network-LlamaForCausalLM/lm_head-Linear/MatMul-op0,Default/network-LlamaForCausalLM/lm_head-Linear/MatMul-op0,kernel_graph0,{18912},18912,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18909,18911,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18658,128_Default/network-LlamaForCausalLM/lm_head-Linear/MatMul-op0,Default/network-LlamaForCausalLM/lm_head-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18912,18921,0x12c28b210e00,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18659,129_Default/network-LlamaForCausalLM/Cast-op0,Default/network-LlamaForCausalLM/Cast-op0,kernel_graph0,{18916-18919},18919,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18915,18922,0x12c187941c00,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18660,130_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/ReduceMax-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/ReduceMax-op0,kernel_graph0,{18919},18919,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18916,18918,0x12c180009200,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18660,130_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/ReduceMax-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/ReduceMax-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18919,18955,0x12c2c9a11000,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18660,131_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Sub-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Sub-op0,kernel_graph0,{18925-18953},18953,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18923,18936,0x12c180009200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18661,132_Default/Mul-op0,Default/Mul-op0,kernel_graph0,{18934},18934,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18925,18933,0x12c26be10c00,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18662,133_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Exp-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Exp-op0,kernel_graph0,{18927},18927,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18927,18952,0x12c187941c00,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18662,134_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/ReduceSum-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/ReduceSum-op0,kernel_graph0,{18950},18950,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18929,18931,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,18929,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18930,18932,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,18930,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18934,18940,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18664,135_Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_RealDiv/RealDiv-op0,Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_RealDiv/RealDiv-op0,kernel_graph0,{18938},18938,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18937,18945,0x12c180009200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18665,136_Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_RealDiv/SumExt-op0,Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_RealDiv/SumExt-op0,kernel_graph0,{18941-18943},18943,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18938,18941,0x12c180009600,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18665,136_Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_RealDiv/SumExt-op0,Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_RealDiv/SumExt-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18943,18948,0x12c187949e00,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18667,138_Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_ReduceSum/Tile-op0,Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_ReduceSum/Tile-op0,kernel_graph0,{18946},18946,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18946,18972,0x12c187952000,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18668,139_Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_Mul/Mul-op0,Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_Mul/Mul-op0,kernel_graph0,{18948-18969},18969,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18950,18956,0x12c187949e00,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18671,141_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Log-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Log-op0,kernel_graph0,{18953},18953,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18953,18978,0x12c26be10c00,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18672,142_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Sub-op1,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Sub-op1,kernel_graph0,{18964-18976},18976,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18957,18963,0x12c187941c00,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18672,143_Default/network-LlamaForCausalLM/StridedSlice-op1,Default/network-LlamaForCausalLM/StridedSlice-op1,kernel_graph0,{18959-18961},18961,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18961,18979,0x12c2aa610e00,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18675,145_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/OneHot-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/OneHot-op0,kernel_graph0,{18966-18976},18976,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18964,18968,0x12c2e8e11000,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18677,146_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Exp-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Exp-op0,kernel_graph0,{18966},18966,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18966,18971,0x12c327611200,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18677,147_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Sub-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Sub-op0,kernel_graph0,{18969},18969,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18969,18975,0x12c2e8e11000,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18678,148_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Mul-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Mul-op0,kernel_graph0,{18973},18973,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18973,19016,0x12c327611200,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18680,149_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op0,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op0,kernel_graph0,{19006-19014},19014,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18976,18982,0x12c2e8e11000,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18681,150_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Mul-op1,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Mul-op1,kernel_graph0,{18980},18980,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18980,18989,0x12c26be10c00,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18682,151_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Neg-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Neg-op0,kernel_graph0,{18983},18983,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18983,18992,0x12c187941c00,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18683,152_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/ReduceSum-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/ReduceSum-op0,kernel_graph0,{18990},18990,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18985,18987,0x12c180009200,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,18985,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18986,18988,0x12c180009600,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,18986,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18990,19000,0x12c187949e00,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18688,153_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Mul-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Mul-op0,kernel_graph0,{18994},18994,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18994,19003,0x12c180009200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18689,154_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/ReduceSum-op1,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/ReduceSum-op1,kernel_graph0,{19001},19001,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18996,18998,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,18996,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 18997,18999,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,18997,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19001,20837,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18689,155_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/RealDiv-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/RealDiv-op0,kernel_graph0,{20835},20835,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19005,19012,0x12c346a11400,0,DefaultEnhancedAscendMemoryPool,262144512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18689,156_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op29,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op29,kernel_graph0,{19010},19010,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19006,19009,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18689,156_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op29,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op29,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19010,19591,0x12c26be10c00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18691,157_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op29,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op29,kernel_graph0,{19589},19589,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19013,19022,0x12c254410600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18692,158_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op0,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op0,kernel_graph0,{19018-19020},19020,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19014,19018,0x12c346a11400,0,DefaultEnhancedAscendMemoryPool,155190272,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18692,158_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op0,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19020,19030,0x12c25c410a00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18694,160_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op1,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op1,kernel_graph0,{19028},19028,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19023,19031,0x12c28b210e00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18694,161_Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Cast-op2,kernel_graph0,{19028},19028,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19026,19036,0x12c254410600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18695,162_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op0,kernel_graph0,{19034},19034,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19027,19606,0x12c18000c000,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18695,162_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op0,kernel_graph0,{19604},19604,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19028,19033,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,16778752,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18695,162_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19034,19123,0x12c25c410800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18696,163_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op0,kernel_graph0,{19038-19044-19052-19120},19120,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19037,19042,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18697,164_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Mul-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Mul-op0,kernel_graph0,{19040},19040,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19040,19806,0x12c180009200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18699,166_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Maximum-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Maximum-op0,kernel_graph0,{19804},19804,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19043,19050,0x12c254410600,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18700,167_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op27,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op27,kernel_graph0,{19048},19048,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19044,19047,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18700,167_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op27,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op27,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19048,19585,0x12c244210200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18701,168_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op27,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op27,kernel_graph0,{19583},19583,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19051,19064,0x12c260410a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18701,169_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op1,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op1,kernel_graph0,{19055-19057-19061},19061,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19052,19055,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18701,169_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op1,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19057,19080,0x12c24ee10400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18703,171_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op0,kernel_graph0,{19059-19071-19078},19078,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19061,19068,0x12c239610000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18706,173_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op1,kernel_graph0,{19064-19066},19066,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19066,19095,0x12c21e80fa00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18707,175_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/Grad_SiLU/SiLUGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/Grad_SiLU/SiLUGrad-op0,kernel_graph0,{19085-19093},19093,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19070,19076,0x12c260410a00,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18707,176_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op28,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op28,kernel_graph0,{19074},19074,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19071,19073,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18707,176_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op28,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op28,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19074,19588,0x12c22ea0fe00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18710,177_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op28,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op28,kernel_graph0,{19586},19586,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19077,19101,0x12c239610000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18711,178_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op2,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op2,kernel_graph0,{19082-19099},19099,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19078,19082,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18711,178_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op2,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19084,19091,0x12c21920f800,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18713,180_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op26,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op26,kernel_graph0,{19089},19089,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19085,19088,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18713,180_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op26,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op26,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19089,19582,0x12c260410a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18714,181_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op26,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op26,kernel_graph0,{19580},19580,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19092,19102,0x12c21120f400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18715,182_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op3,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op3,kernel_graph0,{19097-19099},19099,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19093,19097,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18715,182_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op3,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op3,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19099,19105,0x12c23d610200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18717,184_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/AddN-op0,kernel_graph0,{19103},19103,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19103,19113,0x12c24ee10400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18718,185_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op2,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op2,kernel_graph0,{19111},19111,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19106,19114,0x12c239610000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18719,186_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op2,kernel_graph0,{19111},19111,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19109,19119,0x12c21120f400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18719,187_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op1,kernel_graph0,{19117},19117,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19110,19600,0x12c180010200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18719,187_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op1,kernel_graph0,{19598},19598,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19111,19116,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,16778752,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18719,187_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19117,19122,0x12c239610000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18721,188_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op1,kernel_graph0,{19120},19120,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19120,19287,0x12c23d610200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18722,189_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/AddN-op0,kernel_graph0,{19125-19131-19139-19284},19284,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19124,19129,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18722,190_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Sub-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Sub-op0,kernel_graph0,{19127},19127,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19127,19829,0x12c180009600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18725,192_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/LessEqual-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/LessEqual-op0,kernel_graph0,{19809-19827},19827,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19130,19137,0x12c241610400,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18726,193_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op25,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op25,kernel_graph0,{19135},19135,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19131,19134,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18726,193_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op25,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op25,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19135,19579,0x12c20b20f000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18726,194_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op25,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op25,kernel_graph0,{19577},19577,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19138,19147,0x12c239610000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18728,195_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op4,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op4,kernel_graph0,{19142-19145},19145,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19139,19142,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18728,195_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op4,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op4,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19144,19158,0x12c24ee10400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18729,197_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op0,kernel_graph0,{19153},19153,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19145,19148,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18729,197_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19149,19170,0x12c239610000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18731,198_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,kernel_graph0,{19168},19168,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19150,19167,0x12c252e10600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18731,198_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,kernel_graph0,{19165},19165,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19151,19174,0x12c256e10800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18731,198_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,kernel_graph0,{19172},19172,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19152,19163,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18731,198_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19153,19164,0x12c20f20f200,0,DefaultEnhancedAscendMemoryPool,515969024,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18731,198_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19165,19181,0x12c1f920e600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18732,199_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op5,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op5,kernel_graph0,{19177-19179},19179,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19168,19202,0x12c20320ec00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18732,200_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op3,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op3,kernel_graph0,{19198-19200},19200,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19171,19229,0x12c239610000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18732,201_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op3,kernel_graph0,{19175-19220-19227},19227,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19172,19175,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18732,201_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op3,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19177,19187,0x12c24ee10400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18734,203_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op4,kernel_graph0,{19185},19185,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19179,19184,0x12c256e10600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18735,204_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op5,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op5,kernel_graph0,{19182},19182,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19182,19188,0x12c1f920e600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18735,205_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op1,kernel_graph0,{19185},19185,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19185,19191,0x12c256e10600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18735,206_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op1,kernel_graph0,{19189},19189,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19189,19195,0x12c24ee10400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18736,207_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op6,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op6,kernel_graph0,{19193},19193,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19192,19243,0x12c1f920e600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18737,208_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op2,kernel_graph0,{19196-19234-19241},19241,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19193,19196,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18737,208_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19198,19208,0x12c24ee10400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18740,210_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op2,kernel_graph0,{19206},19206,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19200,19205,0x12c256e10600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18740,211_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op3,kernel_graph0,{19203},19203,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19203,19209,0x12c20320ec00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18741,212_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op0,kernel_graph0,{19206},19206,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19206,19212,0x12c256e10600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18742,213_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op0,kernel_graph0,{19210},19210,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19210,19216,0x12c1fd20e800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18742,214_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op4,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op4,kernel_graph0,{19214},19214,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19213,19258,0x12c20320ec00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18743,215_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op1,kernel_graph0,{19217-19248-19256},19256,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19214,19217,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18743,215_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19219,19225,0x12c241610400,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18745,217_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op24,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op24,kernel_graph0,{19223},19223,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19220,19222,0x12c1ae00d600,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18745,217_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op24,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op24,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19223,19576,0x12c1fd20e800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18746,218_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op24,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op24,kernel_graph0,{19574},19574,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19226,19266,0x12c20720ee00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18747,219_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op7,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op7,kernel_graph0,{19231-19262},19262,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19227,19231,0x12c1ae00d600,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18747,219_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op7,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op7,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19233,19239,0x12c20120ea00,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18749,221_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op23,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op23,kernel_graph0,{19237},19237,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19234,19236,0x12c1ae00d600,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18749,221_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op23,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op23,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19237,19573,0x12c239610000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18751,222_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op23,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op23,kernel_graph0,{19571},19571,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19240,19265,0x12c24ee10400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18752,223_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op6,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op6,kernel_graph0,{19245-19262},19262,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19241,19245,0x12c1ae00d600,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18752,223_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op6,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op6,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19247,19254,0x12c20120ea00,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18755,225_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op22,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op22,kernel_graph0,{19252},19252,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19248,19251,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18755,225_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op22,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op22,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19252,19570,0x12c1ef20e000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18756,226_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op22,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op22,kernel_graph0,{19568},19568,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19255,19264,0x12c1f920e600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18756,227_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op5,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op5,kernel_graph0,{19260-19262},19262,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19256,19260,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18756,227_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op5,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op5,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19262,19269,0x12c20120ea00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18758,229_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/AddN-op0,kernel_graph0,{19267},19267,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19267,19277,0x12c24ee10400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18759,230_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op7,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op7,kernel_graph0,{19275},19275,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19270,19278,0x12c256e10600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18760,231_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op2,kernel_graph0,{19275},19275,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19273,19283,0x12c1f320e200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18762,232_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op2,kernel_graph0,{19281},19281,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19274,19603,0x12c187931800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18762,232_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op2,kernel_graph0,{19601},19601,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19275,19280,0x12c25ee10800,0,DefaultEnhancedAscendMemoryPool,16778752,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18762,232_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19281,19286,0x12c20120ea00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18763,233_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op2,kernel_graph0,{19284},19284,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19284,19369,0x12c20520ec00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18763,234_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/AddN-op0,kernel_graph0,{19287-19290-19298-19366},19366,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19289,19296,0x12c23d610200,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18765,236_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op20,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op20,kernel_graph0,{19294},19294,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19290,19293,0x12c242c10400,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18765,236_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op20,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op20,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19294,19564,0x12c1df00dc00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18767,237_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op20,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op20,kernel_graph0,{19562},19562,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19297,19310,0x12c24ee10400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18768,238_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op8,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op8,kernel_graph0,{19301-19303-19307},19307,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19298,19301,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18768,238_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op8,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op8,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19303,19326,0x12c20f20f200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18771,240_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op6,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op6,kernel_graph0,{19305-19317-19324},19324,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19307,19314,0x12c1d440da00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18772,242_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op7,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op7,kernel_graph0,{19310-19312},19312,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19312,19341,0x12c1b960d400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18775,244_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/Grad_SiLU/SiLUGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/Grad_SiLU/SiLUGrad-op1,kernel_graph0,{19331-19339},19339,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19316,19322,0x12c1e9c0de00,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18775,245_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op21,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op21,kernel_graph0,{19320},19320,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19317,19319,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18775,245_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op21,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op21,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19320,19567,0x12c24ee10400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18777,246_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op21,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op21,kernel_graph0,{19565},19565,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19323,19347,0x12c20120ea00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18778,247_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op9,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op9,kernel_graph0,{19328-19345},19345,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19324,19328,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18778,247_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op9,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op9,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19330,19337,0x12c1b400d200,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18779,249_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op19,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op19,kernel_graph0,{19335},19335,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19331,19334,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18779,249_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op19,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op19,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19335,19561,0x12c1c980d800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18780,250_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op19,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op19,kernel_graph0,{19559},19559,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19338,19348,0x12c1a400d000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18781,251_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op10,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op10,kernel_graph0,{19343-19345},19345,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19339,19343,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18781,251_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op10,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op10,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19345,19351,0x12c1e9c0de00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18783,253_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/AddN-op0,kernel_graph0,{19349},19349,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19349,19359,0x12c1f320e200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18785,254_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op8,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op8,kernel_graph0,{19357},19357,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19352,19360,0x12c1d440da00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18786,255_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op2,kernel_graph0,{19357},19357,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19355,19365,0x12c1a400d000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18787,256_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op3,kernel_graph0,{19363},19363,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19356,19594,0x12c187929600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18787,256_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op3,kernel_graph0,{19592},19592,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19357,19362,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,16778752,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18787,256_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op3,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19363,19368,0x12c20120ea00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18789,257_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op3,kernel_graph0,{19366},19366,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19366,19531,0x12c1e9c0de00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18790,258_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/AddN-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/AddN-op1,kernel_graph0,{19369-19372-19380-19528},19528,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19371,19378,0x12c259a10600,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18791,260_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op18,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op18,kernel_graph0,{19376},19376,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19372,19375,0x12c1edc0e000,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18791,260_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op18,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op18,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19376,19558,0x12c25ba10800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18792,261_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op18,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op18,kernel_graph0,{19556},19556,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19379,19388,0x12c23d610200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18792,262_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op11,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op11,kernel_graph0,{19383-19386},19386,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19380,19383,0x12c1edc0e000,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18792,262_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op11,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op11,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19385,19399,0x12c1f320e200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18795,264_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op4,kernel_graph0,{19394},19394,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19386,19389,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18795,264_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op4,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19390,19412,0x12c1f720e400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18795,265_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,kernel_graph0,{19410},19410,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19391,19409,0x12c23d610200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18795,265_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,kernel_graph0,{19407},19407,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19392,19416,0x12c20120ea00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18795,265_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,kernel_graph0,{19414},19414,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19393,19405,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18795,265_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19394,19406,0x12c20f20f200,0,DefaultEnhancedAscendMemoryPool,515969024,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18795,265_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19407,19423,0x12c18e00c600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18797,266_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op11,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op11,kernel_graph0,{19419-19421},19421,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19410,19446,0x12c1d440da00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18798,267_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op9,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op9,kernel_graph0,{19440-19443},19443,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19413,19473,0x12c20520ec00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18801,268_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op7,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op7,kernel_graph0,{19417-19464-19471},19471,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19414,19417,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18801,268_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op7,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op7,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19419,19429,0x12c1f320e200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18802,270_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op10,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op10,kernel_graph0,{19427},19427,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19421,19426,0x12c20f20f200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18804,271_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op11,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op11,kernel_graph0,{19424},19424,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19424,19430,0x12c18e00c600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18804,272_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op3,kernel_graph0,{19427},19427,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19427,19433,0x12c20f20f200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18805,273_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op1,kernel_graph0,{19431},19431,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19431,19437,0x12c20120ea00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18805,274_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op12,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op12,kernel_graph0,{19435},19435,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19434,19487,0x12c23d610200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18806,275_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op6,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op6,kernel_graph0,{19438-19478-19485},19485,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19435,19438,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18806,275_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op6,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op6,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19440,19452,0x12c18e00c600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18807,277_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op8,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op8,kernel_graph0,{19450},19450,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19443,19449,0x12c1f320e200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18809,278_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op9,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op9,kernel_graph0,{19447},19447,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19447,19453,0x12c1d440da00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18810,279_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op2,kernel_graph0,{19450},19450,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19450,19456,0x12c1f320e200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18810,280_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op0,kernel_graph0,{19454},19454,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19454,19460,0x12c20120ea00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18812,281_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op10,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op10,kernel_graph0,{19458},19458,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19457,19502,0x12c18e00c600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18813,282_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op5,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op5,kernel_graph0,{19461-19492-19500},19500,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19458,19461,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18813,282_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op5,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op5,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19463,19469,0x12c20920ee00,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18816,284_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op17,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op17,kernel_graph0,{19467},19467,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19464,19466,0x12c1edc0e000,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18816,284_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op17,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op17,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19467,19555,0x12c20120ea00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18817,285_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op17,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op17,kernel_graph0,{19553},19553,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19470,19510,0x12c1f320e200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18817,286_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op14,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op14,kernel_graph0,{19475-19506},19506,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19471,19475,0x12c1edc0e000,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18817,286_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op14,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op14,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19477,19483,0x12c259a10600,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18820,288_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op16,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op16,kernel_graph0,{19481},19481,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19478,19480,0x12c1edc0e000,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18820,288_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op16,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op16,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19481,19552,0x12c1f720e400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18820,289_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op16,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op16,kernel_graph0,{19550},19550,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19484,19509,0x12c20520ec00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18822,290_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op13,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op13,kernel_graph0,{19489-19506},19506,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19485,19489,0x12c1edc0e000,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18822,290_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op13,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op13,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19491,19498,0x12c1fb20e600,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18823,292_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op15,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op15,kernel_graph0,{19496},19496,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19492,19495,0x12c1edc0e000,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18823,292_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op15,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op15,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19496,19549,0x12c23d610200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18824,293_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op15,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op15,kernel_graph0,{19547},19547,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19499,19508,0x12c1800a0800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18825,294_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op12,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op12,kernel_graph0,{19504-19506},19506,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19500,19504,0x12c1edc0e000,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18825,294_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op12,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op12,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19506,19513,0x12c1d440da00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18828,296_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/AddN-op0,kernel_graph0,{19511},19511,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19511,19521,0x12c20f20f200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18828,297_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op13,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op13,kernel_graph0,{19519},19519,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19514,19522,0x12c1d440da00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18833,298_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op2,kernel_graph0,{19519},19519,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19517,19527,0x12c21720f400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18835,299_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op4,kernel_graph0,{19525},19525,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19518,19597,0x12c180014400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18835,299_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op4,kernel_graph0,{19595},19595,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19519,19524,0x12c1edc0e000,0,DefaultEnhancedAscendMemoryPool,16778752,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18835,299_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op4,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19525,19530,0x12c1f320e200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18836,300_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op4,kernel_graph0,{19528},19528,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19528,19534,0x12c20520ec00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18836,301_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/AddN-op0,kernel_graph0,{19532},19532,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19532,19537,0x12c1d440da00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18837,302_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op14,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op14,kernel_graph0,{19535},19535,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19535,19546,0x12c20f20f200,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18839,303_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/tok_embeddings-LlamaEmbedding/Grad_Gather/UnsortedSegmentSum-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/tok_embeddings-LlamaEmbedding/Grad_Gather/UnsortedSegmentSum-op0,kernel_graph0,{19544},19544,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19539,19543,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18840,304_Default/Mul-op1,Default/Mul-op1,kernel_graph0,{19541},19541,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19541,19607,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18840,305_Default/Reciprocal-op0,Default/Reciprocal-op0,kernel_graph0,{19544-19547-19550-19553-19556-19559-19562-19565-19568-19571-19574-19577-19580-19583-19586-19589-19592-19595-19598-19601-19604},19604,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19544,19842,0x12c187935a00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18841,306_Default/Mul-op2,Default/Mul-op2,kernel_graph0,{19608-19768-19840},19840,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19547,19845,0x12c1f320e200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18842,307_Default/Mul-op3,Default/Mul-op3,kernel_graph0,{19616-19768-19843},19843,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19550,19848,0x12c1e9c0de00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18843,308_Default/Mul-op4,Default/Mul-op4,kernel_graph0,{19624-19768-19846},19846,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19553,19851,0x12c1f720e400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18844,309_Default/Mul-op5,Default/Mul-op5,kernel_graph0,{19632-19768-19849},19849,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19556,19854,0x12c23d610200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18845,310_Default/Mul-op6,Default/Mul-op6,kernel_graph0,{19640-19768-19852},19852,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19559,19857,0x12c1d440da00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18846,311_Default/Mul-op7,Default/Mul-op7,kernel_graph0,{19648-19768-19855},19855,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19562,19860,0x12c20f20f200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18847,312_Default/Mul-op8,Default/Mul-op8,kernel_graph0,{19656-19768-19858},19858,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19565,19863,0x12c1df00dc00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18847,313_Default/Mul-op9,Default/Mul-op9,kernel_graph0,{19664-19768-19861},19861,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19568,19866,0x12c180018600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18848,314_Default/Mul-op10,Default/Mul-op10,kernel_graph0,{19672-19768-19864},19864,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19571,19869,0x12c1edc0e000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18849,315_Default/Mul-op11,Default/Mul-op11,kernel_graph0,{19680-19768-19867},19867,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19574,19872,0x12c239610000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18850,316_Default/Mul-op12,Default/Mul-op12,kernel_graph0,{19688-19768-19870},19870,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19577,19875,0x12c1fb20e600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18851,317_Default/Mul-op13,Default/Mul-op13,kernel_graph0,{19696-19768-19873},19873,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19580,19878,0x12c1ff20e800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18851,318_Default/Mul-op14,Default/Mul-op14,kernel_graph0,{19704-19768-19876},19876,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19583,19881,0x12c219e0f400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18853,319_Default/Mul-op15,Default/Mul-op15,kernel_graph0,{19712-19768-19879},19879,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19586,19884,0x12c241610400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18854,320_Default/Mul-op16,Default/Mul-op16,kernel_graph0,{19720-19768-19882},19882,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19589,19887,0x12c24c210600,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18854,321_Default/Mul-op17,Default/Mul-op17,kernel_graph0,{19728-19768-19885},19885,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19592,19890,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18857,322_Default/Mul-op18,Default/Mul-op18,kernel_graph0,{19736-19768-19888},19888,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19595,19893,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18858,323_Default/Mul-op19,Default/Mul-op19,kernel_graph0,{19744-19768-19891},19891,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19598,19896,0x12c180014400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18858,324_Default/Mul-op20,Default/Mul-op20,kernel_graph0,{19752-19768-19894},19894,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19601,19899,0x12c180010200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18861,325_Default/Mul-op21,Default/Mul-op21,kernel_graph0,{19760-19768-19897},19897,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19604,19902,0x12c1f1c0e200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18862,326_Default/Mul-op22,Default/Mul-op22,kernel_graph0,{19768-19770-19900},19900,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19608,19613,0x12c1a6d35c00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18862,327_Default/clip_grad_norm-ClipGradNorm/Square-op0,Default/clip_grad_norm-ClipGradNorm/Square-op0,kernel_graph0,{19611},19611,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19610,19780,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18863,328_Default/clip_grad_norm-ClipGradNorm/SumExt-op0,Default/clip_grad_norm-ClipGradNorm/SumExt-op0,kernel_graph0,{19614-19778},19778,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19611,19614,0x12c18000c000,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18863,328_Default/clip_grad_norm-ClipGradNorm/SumExt-op0,Default/clip_grad_norm-ClipGradNorm/SumExt-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19616,19621,0x12c209e0ea00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18865,330_Default/clip_grad_norm-ClipGradNorm/Square-op1,Default/clip_grad_norm-ClipGradNorm/Square-op1,kernel_graph0,{19619},19619,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19618,19781,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18866,331_Default/clip_grad_norm-ClipGradNorm/SumExt-op1,Default/clip_grad_norm-ClipGradNorm/SumExt-op1,kernel_graph0,{19622-19778},19778,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19619,19622,0x12c18000c000,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18866,331_Default/clip_grad_norm-ClipGradNorm/SumExt-op1,Default/clip_grad_norm-ClipGradNorm/SumExt-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19624,19629,0x12c209e0ea00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18868,333_Default/clip_grad_norm-ClipGradNorm/Square-op2,Default/clip_grad_norm-ClipGradNorm/Square-op2,kernel_graph0,{19627},19627,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19626,19782,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18869,334_Default/clip_grad_norm-ClipGradNorm/SumExt-op2,Default/clip_grad_norm-ClipGradNorm/SumExt-op2,kernel_graph0,{19630-19778},19778,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19627,19630,0x12c18000c000,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18869,334_Default/clip_grad_norm-ClipGradNorm/SumExt-op2,Default/clip_grad_norm-ClipGradNorm/SumExt-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19632,19637,0x12c209e0ea00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18871,336_Default/clip_grad_norm-ClipGradNorm/Square-op3,Default/clip_grad_norm-ClipGradNorm/Square-op3,kernel_graph0,{19635},19635,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19634,19783,0x12c180009a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18872,337_Default/clip_grad_norm-ClipGradNorm/SumExt-op3,Default/clip_grad_norm-ClipGradNorm/SumExt-op3,kernel_graph0,{19638-19778},19778,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19635,19638,0x12c18000c000,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18872,337_Default/clip_grad_norm-ClipGradNorm/SumExt-op3,Default/clip_grad_norm-ClipGradNorm/SumExt-op3,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19640,19645,0x12c209e0ea00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18873,339_Default/clip_grad_norm-ClipGradNorm/Square-op4,Default/clip_grad_norm-ClipGradNorm/Square-op4,kernel_graph0,{19643},19643,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19642,19784,0x12c180009c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18875,340_Default/clip_grad_norm-ClipGradNorm/SumExt-op4,Default/clip_grad_norm-ClipGradNorm/SumExt-op4,kernel_graph0,{19646-19778},19778,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19643,19646,0x12c18000c000,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18875,340_Default/clip_grad_norm-ClipGradNorm/SumExt-op4,Default/clip_grad_norm-ClipGradNorm/SumExt-op4,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19648,19653,0x12c224a0f600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18876,342_Default/clip_grad_norm-ClipGradNorm/Square-op5,Default/clip_grad_norm-ClipGradNorm/Square-op5,kernel_graph0,{19651},19651,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19650,19785,0x12c180009e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18878,343_Default/clip_grad_norm-ClipGradNorm/SumExt-op5,Default/clip_grad_norm-ClipGradNorm/SumExt-op5,kernel_graph0,{19654-19778},19778,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19651,19654,0x12c18000c000,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18878,343_Default/clip_grad_norm-ClipGradNorm/SumExt-op5,Default/clip_grad_norm-ClipGradNorm/SumExt-op5,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19656,19661,0x12c224a0f600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18879,345_Default/clip_grad_norm-ClipGradNorm/Square-op6,Default/clip_grad_norm-ClipGradNorm/Square-op6,kernel_graph0,{19659},19659,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19658,19786,0x12c18000a000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18879,346_Default/clip_grad_norm-ClipGradNorm/SumExt-op6,Default/clip_grad_norm-ClipGradNorm/SumExt-op6,kernel_graph0,{19662-19778},19778,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19659,19662,0x12c18000c000,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18879,346_Default/clip_grad_norm-ClipGradNorm/SumExt-op6,Default/clip_grad_norm-ClipGradNorm/SumExt-op6,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19664,19669,0x12c224a0f600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18881,348_Default/clip_grad_norm-ClipGradNorm/Square-op7,Default/clip_grad_norm-ClipGradNorm/Square-op7,kernel_graph0,{19667},19667,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19666,19787,0x12c18000a200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18882,349_Default/clip_grad_norm-ClipGradNorm/SumExt-op7,Default/clip_grad_norm-ClipGradNorm/SumExt-op7,kernel_graph0,{19670-19778},19778,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19667,19670,0x12c18000c000,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18882,349_Default/clip_grad_norm-ClipGradNorm/SumExt-op7,Default/clip_grad_norm-ClipGradNorm/SumExt-op7,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19672,19677,0x12c209e0ea00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18883,351_Default/clip_grad_norm-ClipGradNorm/Square-op8,Default/clip_grad_norm-ClipGradNorm/Square-op8,kernel_graph0,{19675},19675,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19674,19788,0x12c18000a400,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18884,352_Default/clip_grad_norm-ClipGradNorm/SumExt-op8,Default/clip_grad_norm-ClipGradNorm/SumExt-op8,kernel_graph0,{19678-19778},19778,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19675,19678,0x12c18000c000,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18884,352_Default/clip_grad_norm-ClipGradNorm/SumExt-op8,Default/clip_grad_norm-ClipGradNorm/SumExt-op8,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19680,19685,0x12c209e0ea00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18887,354_Default/clip_grad_norm-ClipGradNorm/Square-op9,Default/clip_grad_norm-ClipGradNorm/Square-op9,kernel_graph0,{19683},19683,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19682,19789,0x12c18000a600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18887,355_Default/clip_grad_norm-ClipGradNorm/SumExt-op9,Default/clip_grad_norm-ClipGradNorm/SumExt-op9,kernel_graph0,{19686-19778},19778,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19683,19686,0x12c18000c000,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18887,355_Default/clip_grad_norm-ClipGradNorm/SumExt-op9,Default/clip_grad_norm-ClipGradNorm/SumExt-op9,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19688,19693,0x12c209e0ea00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18889,357_Default/clip_grad_norm-ClipGradNorm/Square-op10,Default/clip_grad_norm-ClipGradNorm/Square-op10,kernel_graph0,{19691},19691,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19690,19790,0x12c18000a800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18890,358_Default/clip_grad_norm-ClipGradNorm/SumExt-op10,Default/clip_grad_norm-ClipGradNorm/SumExt-op10,kernel_graph0,{19694-19778},19778,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19691,19694,0x12c18000c000,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18890,358_Default/clip_grad_norm-ClipGradNorm/SumExt-op10,Default/clip_grad_norm-ClipGradNorm/SumExt-op10,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19696,19701,0x12c209e0ea00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18892,360_Default/clip_grad_norm-ClipGradNorm/Square-op11,Default/clip_grad_norm-ClipGradNorm/Square-op11,kernel_graph0,{19699},19699,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19698,19791,0x12c18000aa00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18892,361_Default/clip_grad_norm-ClipGradNorm/SumExt-op11,Default/clip_grad_norm-ClipGradNorm/SumExt-op11,kernel_graph0,{19702-19778},19778,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19699,19702,0x12c18000c000,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18892,361_Default/clip_grad_norm-ClipGradNorm/SumExt-op11,Default/clip_grad_norm-ClipGradNorm/SumExt-op11,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19704,19709,0x12c224a0f600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18895,363_Default/clip_grad_norm-ClipGradNorm/Square-op12,Default/clip_grad_norm-ClipGradNorm/Square-op12,kernel_graph0,{19707},19707,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19706,19792,0x12c18000ac00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18895,364_Default/clip_grad_norm-ClipGradNorm/SumExt-op12,Default/clip_grad_norm-ClipGradNorm/SumExt-op12,kernel_graph0,{19710-19778},19778,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19707,19710,0x12c18000c000,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18895,364_Default/clip_grad_norm-ClipGradNorm/SumExt-op12,Default/clip_grad_norm-ClipGradNorm/SumExt-op12,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19712,19717,0x12c224a0f600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18899,366_Default/clip_grad_norm-ClipGradNorm/Square-op13,Default/clip_grad_norm-ClipGradNorm/Square-op13,kernel_graph0,{19715},19715,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19714,19793,0x12c18000ae00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18899,367_Default/clip_grad_norm-ClipGradNorm/SumExt-op13,Default/clip_grad_norm-ClipGradNorm/SumExt-op13,kernel_graph0,{19718-19778},19778,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19715,19718,0x12c18000c000,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18899,367_Default/clip_grad_norm-ClipGradNorm/SumExt-op13,Default/clip_grad_norm-ClipGradNorm/SumExt-op13,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19720,19725,0x12c224a0f600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18902,369_Default/clip_grad_norm-ClipGradNorm/Square-op14,Default/clip_grad_norm-ClipGradNorm/Square-op14,kernel_graph0,{19723},19723,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19722,19794,0x12c18000b000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18903,370_Default/clip_grad_norm-ClipGradNorm/SumExt-op14,Default/clip_grad_norm-ClipGradNorm/SumExt-op14,kernel_graph0,{19726-19778},19778,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19723,19726,0x12c18000c000,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18903,370_Default/clip_grad_norm-ClipGradNorm/SumExt-op14,Default/clip_grad_norm-ClipGradNorm/SumExt-op14,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19728,19733,0x12c1a6d35c00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18905,372_Default/clip_grad_norm-ClipGradNorm/Square-op15,Default/clip_grad_norm-ClipGradNorm/Square-op15,kernel_graph0,{19731},19731,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19730,19795,0x12c18000b200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18906,373_Default/clip_grad_norm-ClipGradNorm/SumExt-op15,Default/clip_grad_norm-ClipGradNorm/SumExt-op15,kernel_graph0,{19734-19778},19778,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19731,19734,0x12c18000c000,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18906,373_Default/clip_grad_norm-ClipGradNorm/SumExt-op15,Default/clip_grad_norm-ClipGradNorm/SumExt-op15,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19736,19741,0x12c18000c000,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18907,375_Default/clip_grad_norm-ClipGradNorm/Square-op16,Default/clip_grad_norm-ClipGradNorm/Square-op16,kernel_graph0,{19739},19739,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19738,19796,0x12c18000b400,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18909,376_Default/clip_grad_norm-ClipGradNorm/SumExt-op16,Default/clip_grad_norm-ClipGradNorm/SumExt-op16,kernel_graph0,{19742-19778},19778,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19739,19742,0x12c18000b600,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18909,376_Default/clip_grad_norm-ClipGradNorm/SumExt-op16,Default/clip_grad_norm-ClipGradNorm/SumExt-op16,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19744,19749,0x12c18000c000,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18910,378_Default/clip_grad_norm-ClipGradNorm/Square-op17,Default/clip_grad_norm-ClipGradNorm/Square-op17,kernel_graph0,{19747},19747,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19746,19797,0x12c18000b600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18912,379_Default/clip_grad_norm-ClipGradNorm/SumExt-op17,Default/clip_grad_norm-ClipGradNorm/SumExt-op17,kernel_graph0,{19750-19778},19778,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19747,19750,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18912,379_Default/clip_grad_norm-ClipGradNorm/SumExt-op17,Default/clip_grad_norm-ClipGradNorm/SumExt-op17,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19752,19757,0x12c18000c000,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18913,381_Default/clip_grad_norm-ClipGradNorm/Square-op18,Default/clip_grad_norm-ClipGradNorm/Square-op18,kernel_graph0,{19755},19755,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19754,19798,0x12c18000b800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18916,382_Default/clip_grad_norm-ClipGradNorm/SumExt-op18,Default/clip_grad_norm-ClipGradNorm/SumExt-op18,kernel_graph0,{19758-19778},19778,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19755,19758,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18916,382_Default/clip_grad_norm-ClipGradNorm/SumExt-op18,Default/clip_grad_norm-ClipGradNorm/SumExt-op18,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19760,19765,0x12c18000c000,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18917,384_Default/clip_grad_norm-ClipGradNorm/Square-op19,Default/clip_grad_norm-ClipGradNorm/Square-op19,kernel_graph0,{19763},19763,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19762,19799,0x12c18000ba00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18919,385_Default/clip_grad_norm-ClipGradNorm/SumExt-op19,Default/clip_grad_norm-ClipGradNorm/SumExt-op19,kernel_graph0,{19766-19778},19778,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19763,19766,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18919,385_Default/clip_grad_norm-ClipGradNorm/SumExt-op19,Default/clip_grad_norm-ClipGradNorm/SumExt-op19,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19768,23133,0x12c18000c000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18920,387_Default/AllFinite-op0,Default/AllFinite-op0,kernel_graph0,{19804-19807-19809},19809,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19770,19775,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18922,388_Default/clip_grad_norm-ClipGradNorm/Square-op20,Default/clip_grad_norm-ClipGradNorm/Square-op20,kernel_graph0,{19773},19773,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19772,19800,0x12c18000c200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18923,389_Default/clip_grad_norm-ClipGradNorm/SumExt-op20,Default/clip_grad_norm-ClipGradNorm/SumExt-op20,kernel_graph0,{19776-19778},19778,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19773,19776,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18923,389_Default/clip_grad_norm-ClipGradNorm/SumExt-op20,Default/clip_grad_norm-ClipGradNorm/SumExt-op20,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19778,19803,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18924,391_Default/clip_grad_norm-ClipGradNorm/AddN-op0,Default/clip_grad_norm-ClipGradNorm/AddN-op0,kernel_graph0,{19801},19801,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19801,23130,0x12c18000c200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18925,392_Default/clip_grad_norm-ClipGradNorm/Sqrt-op0,Default/clip_grad_norm-ClipGradNorm/Sqrt-op0,kernel_graph0,{19811-19813},19813,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19804,19834,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18926,393_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Select-op1,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Select-op1,kernel_graph0,{19825-19830},19830,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19807,19951,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18926,394_Default/LogicalNot-op0,Default/LogicalNot-op0,kernel_graph0,{19827},19827,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19809,19837,0x12c180009200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18927,395_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/LogicalOr-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/LogicalOr-op0,kernel_graph0,{19835},19835,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19811,19815,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18928,396_Default/clip_grad_norm-ClipGradNorm/GreaterEqual-op0,Default/clip_grad_norm-ClipGradNorm/GreaterEqual-op0,kernel_graph0,{19813},19813,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19813,19818,0x12c180009a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18928,397_Default/clip_grad_norm-ClipGradNorm/Select-op0,Default/clip_grad_norm-ClipGradNorm/Select-op0,kernel_graph0,{19816},19816,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19816,19821,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18928,398_Default/learning_rate-CosineWithWarmUpLR/Add-op0,Default/learning_rate-CosineWithWarmUpLR/Add-op0,kernel_graph0,{19819},19819,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19819,19824,0x12c180009a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18928,399_Default/clip_grad_norm-ClipGradNorm/Div-op0,Default/clip_grad_norm-ClipGradNorm/Div-op0,kernel_graph0,{19822},19822,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19822,19903,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18928,400_Default/clip_grad_norm-ClipGradNorm/Minimum-op0,Default/clip_grad_norm-ClipGradNorm/Minimum-op0,kernel_graph0,{19840-19843-19846-19849-19852-19855-19858-19861-19864-19867-19870-19873-19876-19879-19882-19885-19888-19891-19894-19897-19900},19900,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19825,19833,0x12c180009a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18928,401_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Mul-op1,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Mul-op1,kernel_graph0,{19830},19830,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19827,19832,0x12c180009c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18929,402_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/LogicalAnd-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/LogicalAnd-op0,kernel_graph0,{19830},19830,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19830,19909,0x12c180009600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18930,403_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Select-op2,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Select-op2,kernel_graph0,{19907},19907,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19835,19905,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18930,404_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Select-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Select-op0,kernel_graph0,{19903},19903,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19838,19907,0x12c180009200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18932,405_Default/learning_rate-CosineWithWarmUpLR/Add-op1,Default/learning_rate-CosineWithWarmUpLR/Add-op1,kernel_graph0,{19905},19905,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19840,20110,0x12c1a6d35c00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18934,406_Default/clip_grad_norm-ClipGradNorm/Mul-op0,Default/clip_grad_norm-ClipGradNorm/Mul-op0,kernel_graph0,{20106-20108},20108,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19843,20100,0x12c209e0ea00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18935,407_Default/clip_grad_norm-ClipGradNorm/Mul-op1,Default/clip_grad_norm-ClipGradNorm/Mul-op1,kernel_graph0,{20096-20098},20098,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19846,20092,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18935,408_Default/clip_grad_norm-ClipGradNorm/Mul-op2,Default/clip_grad_norm-ClipGradNorm/Mul-op2,kernel_graph0,{20088-20090},20090,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19849,20084,0x12c1e9c0de00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18935,409_Default/clip_grad_norm-ClipGradNorm/Mul-op3,Default/clip_grad_norm-ClipGradNorm/Mul-op3,kernel_graph0,{20080-20082},20082,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19852,20076,0x12c1f5c12600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18935,410_Default/clip_grad_norm-ClipGradNorm/Mul-op4,Default/clip_grad_norm-ClipGradNorm/Mul-op4,kernel_graph0,{20072-20074},20074,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19855,20068,0x12c1c6135e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18935,411_Default/clip_grad_norm-ClipGradNorm/Mul-op5,Default/clip_grad_norm-ClipGradNorm/Mul-op5,kernel_graph0,{20064-20066},20066,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19858,20060,0x12c1d0d36000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18936,412_Default/clip_grad_norm-ClipGradNorm/Mul-op6,Default/clip_grad_norm-ClipGradNorm/Mul-op6,kernel_graph0,{20056-20058},20058,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19861,20052,0x12c20de0ec00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18938,413_Default/clip_grad_norm-ClipGradNorm/Mul-op7,Default/clip_grad_norm-ClipGradNorm/Mul-op7,kernel_graph0,{20048-20050},20050,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19864,20044,0x12c23d610200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18939,414_Default/clip_grad_norm-ClipGradNorm/Mul-op8,Default/clip_grad_norm-ClipGradNorm/Mul-op8,kernel_graph0,{20040-20042},20042,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19867,20036,0x12c1db936200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18939,415_Default/clip_grad_norm-ClipGradNorm/Mul-op9,Default/clip_grad_norm-ClipGradNorm/Mul-op9,kernel_graph0,{20032-20034},20034,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19870,20028,0x12c1edc0e000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18941,416_Default/clip_grad_norm-ClipGradNorm/Mul-op10,Default/clip_grad_norm-ClipGradNorm/Mul-op10,kernel_graph0,{20024-20026},20026,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19873,20020,0x12c1df936400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18942,417_Default/clip_grad_norm-ClipGradNorm/Mul-op11,Default/clip_grad_norm-ClipGradNorm/Mul-op11,kernel_graph0,{20016-20018},20018,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19876,20012,0x12c224a0f600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18943,418_Default/clip_grad_norm-ClipGradNorm/Mul-op12,Default/clip_grad_norm-ClipGradNorm/Mul-op12,kernel_graph0,{20008-20010},20010,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19879,20004,0x12c22f60f800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18944,419_Default/clip_grad_norm-ClipGradNorm/Mul-op13,Default/clip_grad_norm-ClipGradNorm/Mul-op13,kernel_graph0,{20000-20002},20002,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19882,19996,0x12c218a0ee00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18944,420_Default/clip_grad_norm-ClipGradNorm/Mul-op14,Default/clip_grad_norm-ClipGradNorm/Mul-op14,kernel_graph0,{19992-19994},19994,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19885,19988,0x12c180018600,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18946,421_Default/clip_grad_norm-ClipGradNorm/Mul-op15,Default/clip_grad_norm-ClipGradNorm/Mul-op15,kernel_graph0,{19984-19986},19986,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19888,19980,0x12c22360f000,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18947,422_Default/clip_grad_norm-ClipGradNorm/Mul-op16,Default/clip_grad_norm-ClipGradNorm/Mul-op16,kernel_graph0,{19976-19978},19978,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19891,19972,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18947,423_Default/clip_grad_norm-ClipGradNorm/Mul-op17,Default/clip_grad_norm-ClipGradNorm/Mul-op17,kernel_graph0,{19968-19970},19970,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19894,19964,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18948,424_Default/clip_grad_norm-ClipGradNorm/Mul-op18,Default/clip_grad_norm-ClipGradNorm/Mul-op18,kernel_graph0,{19960-19962},19962,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19897,19956,0x12c180014400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18949,425_Default/clip_grad_norm-ClipGradNorm/Mul-op19,Default/clip_grad_norm-ClipGradNorm/Mul-op19,kernel_graph0,{19952-19954},19954,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19900,20118,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18950,426_Default/clip_grad_norm-ClipGradNorm/Mul-op20,Default/clip_grad_norm-ClipGradNorm/Mul-op20,kernel_graph0,{20114-20116},20116,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19910,19919,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18954,430_Default/learning_rate-CosineWithWarmUpLR/Cast-op0,Default/learning_rate-CosineWithWarmUpLR/Cast-op0,kernel_graph0,{19912-19915-19917},19917,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19912,19914,0x12c180009200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,18957,431_Default/learning_rate-CosineWithWarmUpLR/GreaterEqual-op0,Default/learning_rate-CosineWithWarmUpLR/GreaterEqual-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19915,19920,0x12c180009200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19914,435_Default/learning_rate-CosineWithWarmUpLR/Greater-op0,Default/learning_rate-CosineWithWarmUpLR/Greater-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19917,19923,0x12c180009600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19915,436_Default/TensorMove-op0,Default/TensorMove-op0,kernel_graph0,{19921},19921,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19921,19926,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19920,440_Default/learning_rate-CosineWithWarmUpLR/Sub-op1,Default/learning_rate-CosineWithWarmUpLR/Sub-op1,kernel_graph0,{19924},19924,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19924,19929,0x12c180009200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19921,441_Default/learning_rate-CosineWithWarmUpLR/Div-op1,Default/learning_rate-CosineWithWarmUpLR/Div-op1,kernel_graph0,{19927},19927,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19927,19932,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19922,442_Default/learning_rate-CosineWithWarmUpLR/Mul-op1,Default/learning_rate-CosineWithWarmUpLR/Mul-op1,kernel_graph0,{19930},19930,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19930,19935,0x12c180009200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19922,443_Default/learning_rate-CosineWithWarmUpLR/Cos-op0,Default/learning_rate-CosineWithWarmUpLR/Cos-op0,kernel_graph0,{19933},19933,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19933,19938,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19924,444_Default/learning_rate-CosineWithWarmUpLR/Add-op3,Default/learning_rate-CosineWithWarmUpLR/Add-op3,kernel_graph0,{19936},19936,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19936,19941,0x12c180009200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19925,445_Default/learning_rate-CosineWithWarmUpLR/Mul-op2,Default/learning_rate-CosineWithWarmUpLR/Mul-op2,kernel_graph0,{19939},19939,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19939,19944,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19925,446_Default/learning_rate-CosineWithWarmUpLR/Maximum-op0,Default/learning_rate-CosineWithWarmUpLR/Maximum-op0,kernel_graph0,{19942},19942,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19942,19947,0x12c180009200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19927,447_Default/learning_rate-CosineWithWarmUpLR/Mul-op3,Default/learning_rate-CosineWithWarmUpLR/Mul-op3,kernel_graph0,{19945},19945,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19945,19950,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19928,448_Default/learning_rate-CosineWithWarmUpLR/Add-op4,Default/learning_rate-CosineWithWarmUpLR/Add-op4,kernel_graph0,{19948},19948,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19948,23131,0x12c180009200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19931,451_Default/Reshape-op0,Default/Reshape-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19952,20781,0x12c1f1c0e200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19951,453_Default/optimizer-FP32StateAdamWeightDecay/Mul-op96,Default/optimizer-FP32StateAdamWeightDecay/Mul-op96,kernel_graph0,{20778},20778,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19954,19959,0x12c223613200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19952,454_Default/optimizer-FP32StateAdamWeightDecay/Square-op19,Default/optimizer-FP32StateAdamWeightDecay/Square-op19,kernel_graph0,{19957},19957,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19957,20777,0x12c180010600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19953,455_Default/optimizer-FP32StateAdamWeightDecay/Mul-op98,Default/optimizer-FP32StateAdamWeightDecay/Mul-op98,kernel_graph0,{20774},20774,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19960,20745,0x12c223613200,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19953,456_Default/optimizer-FP32StateAdamWeightDecay/Mul-op91,Default/optimizer-FP32StateAdamWeightDecay/Mul-op91,kernel_graph0,{20742},20742,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19962,19967,0x12c223617400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19954,457_Default/optimizer-FP32StateAdamWeightDecay/Square-op18,Default/optimizer-FP32StateAdamWeightDecay/Square-op18,kernel_graph0,{19965},19965,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19965,20749,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19955,458_Default/optimizer-FP32StateAdamWeightDecay/Mul-op93,Default/optimizer-FP32StateAdamWeightDecay/Mul-op93,kernel_graph0,{20746},20746,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19968,20713,0x12c223617400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19955,459_Default/optimizer-FP32StateAdamWeightDecay/Mul-op86,Default/optimizer-FP32StateAdamWeightDecay/Mul-op86,kernel_graph0,{20710},20710,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19970,19975,0x12c22361b600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19957,460_Default/optimizer-FP32StateAdamWeightDecay/Square-op17,Default/optimizer-FP32StateAdamWeightDecay/Square-op17,kernel_graph0,{19973},19973,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19973,20717,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19958,461_Default/optimizer-FP32StateAdamWeightDecay/Mul-op88,Default/optimizer-FP32StateAdamWeightDecay/Mul-op88,kernel_graph0,{20714},20714,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19976,20681,0x12c22361b600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19958,462_Default/optimizer-FP32StateAdamWeightDecay/Mul-op81,Default/optimizer-FP32StateAdamWeightDecay/Mul-op81,kernel_graph0,{20678},20678,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19978,19983,0x12c22361f800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19960,463_Default/optimizer-FP32StateAdamWeightDecay/Square-op16,Default/optimizer-FP32StateAdamWeightDecay/Square-op16,kernel_graph0,{19981},19981,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19981,20685,0x12c22360f000,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19961,464_Default/optimizer-FP32StateAdamWeightDecay/Mul-op83,Default/optimizer-FP32StateAdamWeightDecay/Mul-op83,kernel_graph0,{20682},20682,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19984,20649,0x12c241610400,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19961,465_Default/optimizer-FP32StateAdamWeightDecay/Mul-op76,Default/optimizer-FP32StateAdamWeightDecay/Mul-op76,kernel_graph0,{20646},20646,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19986,19991,0x12c260a10600,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19962,466_Default/optimizer-FP32StateAdamWeightDecay/Square-op15,Default/optimizer-FP32StateAdamWeightDecay/Square-op15,kernel_graph0,{19989},19989,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19989,20653,0x12c180014800,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19963,467_Default/optimizer-FP32StateAdamWeightDecay/Mul-op78,Default/optimizer-FP32StateAdamWeightDecay/Mul-op78,kernel_graph0,{20650},20650,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19992,20617,0x12c1f9c12800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19963,468_Default/optimizer-FP32StateAdamWeightDecay/Mul-op71,Default/optimizer-FP32StateAdamWeightDecay/Mul-op71,kernel_graph0,{20614},20614,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19994,19999,0x12c260a10600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19965,469_Default/optimizer-FP32StateAdamWeightDecay/Square-op14,Default/optimizer-FP32StateAdamWeightDecay/Square-op14,kernel_graph0,{19997},19997,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 19997,20621,0x12c218a0ee00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19966,470_Default/optimizer-FP32StateAdamWeightDecay/Mul-op73,Default/optimizer-FP32StateAdamWeightDecay/Mul-op73,kernel_graph0,{20618},20618,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20000,20585,0x12c260a10600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19966,471_Default/optimizer-FP32StateAdamWeightDecay/Mul-op66,Default/optimizer-FP32StateAdamWeightDecay/Mul-op66,kernel_graph0,{20582},20582,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20002,20007,0x12c26b610800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19968,472_Default/optimizer-FP32StateAdamWeightDecay/Square-op13,Default/optimizer-FP32StateAdamWeightDecay/Square-op13,kernel_graph0,{20005},20005,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20005,20589,0x12c22f60f800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19969,473_Default/optimizer-FP32StateAdamWeightDecay/Mul-op68,Default/optimizer-FP32StateAdamWeightDecay/Mul-op68,kernel_graph0,{20586},20586,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20008,20553,0x12c26b610800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19969,474_Default/optimizer-FP32StateAdamWeightDecay/Mul-op61,Default/optimizer-FP32StateAdamWeightDecay/Mul-op61,kernel_graph0,{20550},20550,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20010,20015,0x12c276210a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19970,475_Default/optimizer-FP32StateAdamWeightDecay/Square-op12,Default/optimizer-FP32StateAdamWeightDecay/Square-op12,kernel_graph0,{20013},20013,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20013,20557,0x12c22361f800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19971,476_Default/optimizer-FP32StateAdamWeightDecay/Mul-op63,Default/optimizer-FP32StateAdamWeightDecay/Mul-op63,kernel_graph0,{20554},20554,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20016,20521,0x12c204812a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19971,477_Default/optimizer-FP32StateAdamWeightDecay/Mul-op56,Default/optimizer-FP32StateAdamWeightDecay/Mul-op56,kernel_graph0,{20518},20518,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20018,20023,0x12c1e3936600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19973,478_Default/optimizer-FP32StateAdamWeightDecay/Square-op11,Default/optimizer-FP32StateAdamWeightDecay/Square-op11,kernel_graph0,{20021},20021,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20021,20525,0x12c1df936400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19974,479_Default/optimizer-FP32StateAdamWeightDecay/Mul-op58,Default/optimizer-FP32StateAdamWeightDecay/Mul-op58,kernel_graph0,{20522},20522,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20024,20489,0x12c1e3936600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19974,480_Default/optimizer-FP32StateAdamWeightDecay/Mul-op51,Default/optimizer-FP32StateAdamWeightDecay/Mul-op51,kernel_graph0,{20486},20486,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20026,20031,0x12c19f414a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19976,481_Default/optimizer-FP32StateAdamWeightDecay/Square-op10,Default/optimizer-FP32StateAdamWeightDecay/Square-op10,kernel_graph0,{20029},20029,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20029,20493,0x12c1edc0e000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19977,482_Default/optimizer-FP32StateAdamWeightDecay/Mul-op53,Default/optimizer-FP32StateAdamWeightDecay/Mul-op53,kernel_graph0,{20490},20490,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20032,20457,0x12c19f414a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19977,483_Default/optimizer-FP32StateAdamWeightDecay/Mul-op46,Default/optimizer-FP32StateAdamWeightDecay/Mul-op46,kernel_graph0,{20454},20454,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20034,20039,0x12c276210a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19978,484_Default/optimizer-FP32StateAdamWeightDecay/Square-op9,Default/optimizer-FP32StateAdamWeightDecay/Square-op9,kernel_graph0,{20037},20037,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20037,20461,0x12c1db936200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19979,485_Default/optimizer-FP32StateAdamWeightDecay/Mul-op48,Default/optimizer-FP32StateAdamWeightDecay/Mul-op48,kernel_graph0,{20458},20458,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20040,20425,0x12c276210a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19981,486_Default/optimizer-FP32StateAdamWeightDecay/Mul-op41,Default/optimizer-FP32StateAdamWeightDecay/Mul-op41,kernel_graph0,{20422},20422,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20042,20047,0x12c27a210c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19982,487_Default/optimizer-FP32StateAdamWeightDecay/Square-op8,Default/optimizer-FP32StateAdamWeightDecay/Square-op8,kernel_graph0,{20045},20045,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20045,20429,0x12c23a20fa00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19982,488_Default/optimizer-FP32StateAdamWeightDecay/Mul-op43,Default/optimizer-FP32StateAdamWeightDecay/Mul-op43,kernel_graph0,{20426},20426,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20048,20393,0x12c27a210c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19984,489_Default/optimizer-FP32StateAdamWeightDecay/Mul-op36,Default/optimizer-FP32StateAdamWeightDecay/Mul-op36,kernel_graph0,{20390},20390,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20050,20055,0x12c284e10e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19985,490_Default/optimizer-FP32StateAdamWeightDecay/Square-op7,Default/optimizer-FP32StateAdamWeightDecay/Square-op7,kernel_graph0,{20053},20053,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20053,20397,0x12c20de0ec00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19985,491_Default/optimizer-FP32StateAdamWeightDecay/Mul-op38,Default/optimizer-FP32StateAdamWeightDecay/Mul-op38,kernel_graph0,{20394},20394,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20056,20361,0x12c284e10e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19986,492_Default/optimizer-FP32StateAdamWeightDecay/Mul-op31,Default/optimizer-FP32StateAdamWeightDecay/Mul-op31,kernel_graph0,{20358},20358,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20058,20063,0x12c28fa11000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19987,493_Default/optimizer-FP32StateAdamWeightDecay/Square-op6,Default/optimizer-FP32StateAdamWeightDecay/Square-op6,kernel_graph0,{20061},20061,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20061,20365,0x12c1d0d36000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19987,494_Default/optimizer-FP32StateAdamWeightDecay/Mul-op33,Default/optimizer-FP32StateAdamWeightDecay/Mul-op33,kernel_graph0,{20362},20362,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20064,20329,0x12c28fa11000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19989,495_Default/optimizer-FP32StateAdamWeightDecay/Mul-op26,Default/optimizer-FP32StateAdamWeightDecay/Mul-op26,kernel_graph0,{20326},20326,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20066,20071,0x12c29a611200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19990,496_Default/optimizer-FP32StateAdamWeightDecay/Square-op5,Default/optimizer-FP32StateAdamWeightDecay/Square-op5,kernel_graph0,{20069},20069,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20069,20333,0x12c1c6135e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19990,497_Default/optimizer-FP32StateAdamWeightDecay/Mul-op28,Default/optimizer-FP32StateAdamWeightDecay/Mul-op28,kernel_graph0,{20330},20330,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20072,20297,0x12c29a611200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19992,498_Default/optimizer-FP32StateAdamWeightDecay/Mul-op21,Default/optimizer-FP32StateAdamWeightDecay/Mul-op21,kernel_graph0,{20294},20294,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20074,20079,0x12c29e611400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19993,499_Default/optimizer-FP32StateAdamWeightDecay/Square-op4,Default/optimizer-FP32StateAdamWeightDecay/Square-op4,kernel_graph0,{20077},20077,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20077,20301,0x12c1f5c12600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19993,500_Default/optimizer-FP32StateAdamWeightDecay/Mul-op23,Default/optimizer-FP32StateAdamWeightDecay/Mul-op23,kernel_graph0,{20298},20298,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20080,20265,0x12c29e611400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19994,501_Default/optimizer-FP32StateAdamWeightDecay/Mul-op16,Default/optimizer-FP32StateAdamWeightDecay/Mul-op16,kernel_graph0,{20262},20262,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20082,20087,0x12c2a2611600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19995,502_Default/optimizer-FP32StateAdamWeightDecay/Square-op3,Default/optimizer-FP32StateAdamWeightDecay/Square-op3,kernel_graph0,{20085},20085,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20085,20269,0x12c1e7936800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19995,503_Default/optimizer-FP32StateAdamWeightDecay/Mul-op18,Default/optimizer-FP32StateAdamWeightDecay/Mul-op18,kernel_graph0,{20266},20266,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20088,20233,0x12c2a2611600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19997,504_Default/optimizer-FP32StateAdamWeightDecay/Mul-op11,Default/optimizer-FP32StateAdamWeightDecay/Mul-op11,kernel_graph0,{20230},20230,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20090,20095,0x12c2a6611800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19998,505_Default/optimizer-FP32StateAdamWeightDecay/Square-op2,Default/optimizer-FP32StateAdamWeightDecay/Square-op2,kernel_graph0,{20093},20093,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20093,20237,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,19998,506_Default/optimizer-FP32StateAdamWeightDecay/Mul-op13,Default/optimizer-FP32StateAdamWeightDecay/Mul-op13,kernel_graph0,{20234},20234,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20096,20201,0x12c2a6611800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20000,507_Default/optimizer-FP32StateAdamWeightDecay/Mul-op6,Default/optimizer-FP32StateAdamWeightDecay/Mul-op6,kernel_graph0,{20198},20198,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20098,20103,0x12c2aa611a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20001,508_Default/optimizer-FP32StateAdamWeightDecay/Square-op1,Default/optimizer-FP32StateAdamWeightDecay/Square-op1,kernel_graph0,{20101},20101,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20101,20205,0x12c208812c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20001,509_Default/optimizer-FP32StateAdamWeightDecay/Mul-op8,Default/optimizer-FP32StateAdamWeightDecay/Mul-op8,kernel_graph0,{20202},20202,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20104,20129,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20002,510_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Cast-op0,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Cast-op0,kernel_graph0,{20122-20125-20127},20127,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20106,20169,0x12c2aa611a00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20003,511_Default/optimizer-FP32StateAdamWeightDecay/Mul-op1,Default/optimizer-FP32StateAdamWeightDecay/Mul-op1,kernel_graph0,{20166},20166,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20108,20113,0x12c2c9a11c00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20003,512_Default/optimizer-FP32StateAdamWeightDecay/Square-op0,Default/optimizer-FP32StateAdamWeightDecay/Square-op0,kernel_graph0,{20111},20111,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20111,20173,0x12c1a3414c00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20005,513_Default/optimizer-FP32StateAdamWeightDecay/Mul-op3,Default/optimizer-FP32StateAdamWeightDecay/Mul-op3,kernel_graph0,{20170},20170,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20114,20813,0x12c22e21fa00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20006,514_Default/optimizer-FP32StateAdamWeightDecay/Mul-op101,Default/optimizer-FP32StateAdamWeightDecay/Mul-op101,kernel_graph0,{20810},20810,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20116,20121,0x12c22e223c00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20006,515_Default/optimizer-FP32StateAdamWeightDecay/Square-op20,Default/optimizer-FP32StateAdamWeightDecay/Square-op20,kernel_graph0,{20119},20119,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20119,20809,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20008,516_Default/optimizer-FP32StateAdamWeightDecay/Mul-op103,Default/optimizer-FP32StateAdamWeightDecay/Mul-op103,kernel_graph0,{20806},20806,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20122,20124,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20009,518_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/GreaterEqual-op0,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/GreaterEqual-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20125,20130,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20124,522_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Greater-op0,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Greater-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20127,20133,0x12c180009600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20125,523_Default/TensorMove-op1,Default/TensorMove-op1,kernel_graph0,{20131},20131,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20131,20136,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20130,527_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Sub-op1,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Sub-op1,kernel_graph0,{20134},20134,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20134,20139,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20131,528_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Div-op1,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Div-op1,kernel_graph0,{20137},20137,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20137,20142,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20132,529_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Mul-op1,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Mul-op1,kernel_graph0,{20140},20140,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20140,20145,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20132,530_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Cos-op0,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Cos-op0,kernel_graph0,{20143},20143,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20143,20148,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20134,531_Default/learning_rate-CosineWithWarmUpLR/Add-op6,Default/learning_rate-CosineWithWarmUpLR/Add-op6,kernel_graph0,{20146},20146,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20146,20151,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20135,532_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Mul-op2,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Mul-op2,kernel_graph0,{20149},20149,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20149,20154,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20135,533_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Maximum-op0,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Maximum-op0,kernel_graph0,{20152},20152,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20152,20157,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20137,534_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Mul-op3,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Mul-op3,kernel_graph0,{20155},20155,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20155,20160,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20138,535_Default/learning_rate-CosineWithWarmUpLR/Add-op7,Default/learning_rate-CosineWithWarmUpLR/Add-op7,kernel_graph0,{20158},20158,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20158,20824,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20141,538_Default/optimizer-FP32StateAdamWeightDecay/Reshape-op0,Default/optimizer-FP32StateAdamWeightDecay/Reshape-op0,kernel_graph0,{20182-20214-20246-20278-20310-20342-20374-20406-20438-20470-20502-20534-20566-20598-20630-20662-20694-20726-20758-20790-20822},20822,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20162,20168,0x12c2c9a11c00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20143,540_Default/optimizer-FP32StateAdamWeightDecay/Mul-op0,Default/optimizer-FP32StateAdamWeightDecay/Mul-op0,kernel_graph0,{20166},20166,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20164,20172,0x12c2e8e11e00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20144,541_Default/optimizer-FP32StateAdamWeightDecay/Mul-op2,Default/optimizer-FP32StateAdamWeightDecay/Mul-op2,kernel_graph0,{20170},20170,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20166,20191,0x12c308212000,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20146,542_Default/learning_rate-CosineWithWarmUpLR/Add-op8,Default/learning_rate-CosineWithWarmUpLR/Add-op8,kernel_graph0,{20179-20189},20189,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20170,20189,0x12c2aa611a00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20146,543_Default/learning_rate-CosineWithWarmUpLR/Add-op9,Default/learning_rate-CosineWithWarmUpLR/Add-op9,kernel_graph0,{20174-20187},20187,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20174,20178,0x12c1a3414c00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20147,544_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op0,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op0,kernel_graph0,{20176},20176,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20176,20181,0x12c2c9a11c00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20148,545_Default/learning_rate-CosineWithWarmUpLR/Add-op10,Default/learning_rate-CosineWithWarmUpLR/Add-op10,kernel_graph0,{20179},20179,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20179,20184,0x12c2e8e11e00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20149,546_Default/optimizer-FP32StateAdamWeightDecay/Div-op0,Default/optimizer-FP32StateAdamWeightDecay/Div-op0,kernel_graph0,{20182},20182,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20182,20187,0x12c2c9a11c00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20150,547_Default/optimizer-FP32StateAdamWeightDecay/Mul-op4,Default/optimizer-FP32StateAdamWeightDecay/Mul-op4,kernel_graph0,{20185},20185,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20185,20193,0x12c2e8e11e00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20151,548_Default/optimizer-FP32StateAdamWeightDecay/Sub-op0,Default/optimizer-FP32StateAdamWeightDecay/Sub-op0,kernel_graph0,{20191},20191,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20194,20200,0x12c1a3414c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20155,552_Default/optimizer-FP32StateAdamWeightDecay/Mul-op5,Default/optimizer-FP32StateAdamWeightDecay/Mul-op5,kernel_graph0,{20198},20198,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20196,20204,0x12c1a7414e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20156,553_Default/optimizer-FP32StateAdamWeightDecay/Mul-op7,Default/optimizer-FP32StateAdamWeightDecay/Mul-op7,kernel_graph0,{20202},20202,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20198,20223,0x12c1ab415000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20157,554_Default/learning_rate-CosineWithWarmUpLR/Add-op11,Default/learning_rate-CosineWithWarmUpLR/Add-op11,kernel_graph0,{20211-20221},20221,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20202,20221,0x12c1a3414c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20158,555_Default/learning_rate-CosineWithWarmUpLR/Add-op12,Default/learning_rate-CosineWithWarmUpLR/Add-op12,kernel_graph0,{20206-20219},20219,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20206,20210,0x12c1a7414e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20159,556_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op1,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op1,kernel_graph0,{20208},20208,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20208,20213,0x12c208812c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20160,557_Default/learning_rate-CosineWithWarmUpLR/Add-op13,Default/learning_rate-CosineWithWarmUpLR/Add-op13,kernel_graph0,{20211},20211,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20211,20216,0x12c1a7414e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20161,558_Default/optimizer-FP32StateAdamWeightDecay/Div-op1,Default/optimizer-FP32StateAdamWeightDecay/Div-op1,kernel_graph0,{20214},20214,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20214,20219,0x12c208812c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20161,559_Default/optimizer-FP32StateAdamWeightDecay/Mul-op9,Default/optimizer-FP32StateAdamWeightDecay/Mul-op9,kernel_graph0,{20217},20217,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20217,20225,0x12c1a7414e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20161,560_Default/optimizer-FP32StateAdamWeightDecay/Sub-op1,Default/optimizer-FP32StateAdamWeightDecay/Sub-op1,kernel_graph0,{20223},20223,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20226,20232,0x12c208812c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20163,564_Default/optimizer-FP32StateAdamWeightDecay/Mul-op10,Default/optimizer-FP32StateAdamWeightDecay/Mul-op10,kernel_graph0,{20230},20230,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20228,20236,0x12c1a3414c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20164,565_Default/optimizer-FP32StateAdamWeightDecay/Mul-op12,Default/optimizer-FP32StateAdamWeightDecay/Mul-op12,kernel_graph0,{20234},20234,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20230,20255,0x12c1a7414e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20164,566_Default/learning_rate-CosineWithWarmUpLR/Add-op14,Default/learning_rate-CosineWithWarmUpLR/Add-op14,kernel_graph0,{20243-20253},20253,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20234,20253,0x12c208812c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20165,567_Default/learning_rate-CosineWithWarmUpLR/Add-op15,Default/learning_rate-CosineWithWarmUpLR/Add-op15,kernel_graph0,{20238-20251},20251,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20238,20242,0x12c1a3414c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20165,568_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op2,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op2,kernel_graph0,{20240},20240,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20240,20245,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20166,569_Default/learning_rate-CosineWithWarmUpLR/Add-op16,Default/learning_rate-CosineWithWarmUpLR/Add-op16,kernel_graph0,{20243},20243,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20243,20248,0x12c1a3414c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20167,570_Default/optimizer-FP32StateAdamWeightDecay/Div-op2,Default/optimizer-FP32StateAdamWeightDecay/Div-op2,kernel_graph0,{20246},20246,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20246,20251,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20169,571_Default/optimizer-FP32StateAdamWeightDecay/Mul-op14,Default/optimizer-FP32StateAdamWeightDecay/Mul-op14,kernel_graph0,{20249},20249,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20249,20257,0x12c1a3414c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20170,572_Default/optimizer-FP32StateAdamWeightDecay/Sub-op2,Default/optimizer-FP32StateAdamWeightDecay/Sub-op2,kernel_graph0,{20255},20255,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20258,20264,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20175,576_Default/optimizer-FP32StateAdamWeightDecay/Mul-op15,Default/optimizer-FP32StateAdamWeightDecay/Mul-op15,kernel_graph0,{20262},20262,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20260,20268,0x12c208812c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20176,577_Default/optimizer-FP32StateAdamWeightDecay/Mul-op17,Default/optimizer-FP32StateAdamWeightDecay/Mul-op17,kernel_graph0,{20266},20266,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20262,20287,0x12c1a3414c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20177,578_Default/learning_rate-CosineWithWarmUpLR/Add-op17,Default/learning_rate-CosineWithWarmUpLR/Add-op17,kernel_graph0,{20275-20285},20285,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20266,20285,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20177,579_Default/learning_rate-CosineWithWarmUpLR/Add-op18,Default/learning_rate-CosineWithWarmUpLR/Add-op18,kernel_graph0,{20270-20283},20283,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20270,20274,0x12c208812c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20179,580_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op3,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op3,kernel_graph0,{20272},20272,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20272,20277,0x12c1e7936800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20180,581_Default/learning_rate-CosineWithWarmUpLR/Add-op19,Default/learning_rate-CosineWithWarmUpLR/Add-op19,kernel_graph0,{20275},20275,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20275,20280,0x12c208812c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20180,582_Default/optimizer-FP32StateAdamWeightDecay/Div-op3,Default/optimizer-FP32StateAdamWeightDecay/Div-op3,kernel_graph0,{20278},20278,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20278,20283,0x12c1e7936800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20182,583_Default/optimizer-FP32StateAdamWeightDecay/Mul-op19,Default/optimizer-FP32StateAdamWeightDecay/Mul-op19,kernel_graph0,{20281},20281,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20281,20289,0x12c208812c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20183,584_Default/optimizer-FP32StateAdamWeightDecay/Sub-op3,Default/optimizer-FP32StateAdamWeightDecay/Sub-op3,kernel_graph0,{20287},20287,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20290,20296,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20186,588_Default/optimizer-FP32StateAdamWeightDecay/Mul-op20,Default/optimizer-FP32StateAdamWeightDecay/Mul-op20,kernel_graph0,{20294},20294,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20292,20300,0x12c208812c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20187,589_Default/optimizer-FP32StateAdamWeightDecay/Mul-op22,Default/optimizer-FP32StateAdamWeightDecay/Mul-op22,kernel_graph0,{20298},20298,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20294,20319,0x12c1e7936800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20188,590_Default/learning_rate-CosineWithWarmUpLR/Add-op20,Default/learning_rate-CosineWithWarmUpLR/Add-op20,kernel_graph0,{20307-20317},20317,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20298,20317,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20188,591_Default/learning_rate-CosineWithWarmUpLR/Add-op21,Default/learning_rate-CosineWithWarmUpLR/Add-op21,kernel_graph0,{20302-20315},20315,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20302,20306,0x12c1f5c12600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20188,592_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op4,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op4,kernel_graph0,{20304},20304,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20304,20309,0x12c208812c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20188,593_Default/learning_rate-CosineWithWarmUpLR/Add-op22,Default/learning_rate-CosineWithWarmUpLR/Add-op22,kernel_graph0,{20307},20307,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20307,20312,0x12c1f5c12600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20189,594_Default/optimizer-FP32StateAdamWeightDecay/Div-op4,Default/optimizer-FP32StateAdamWeightDecay/Div-op4,kernel_graph0,{20310},20310,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20310,20315,0x12c208812c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20190,595_Default/optimizer-FP32StateAdamWeightDecay/Mul-op24,Default/optimizer-FP32StateAdamWeightDecay/Mul-op24,kernel_graph0,{20313},20313,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20313,20321,0x12c1f5c12600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20190,596_Default/optimizer-FP32StateAdamWeightDecay/Sub-op4,Default/optimizer-FP32StateAdamWeightDecay/Sub-op4,kernel_graph0,{20319},20319,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20322,20328,0x12c1a3414c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20192,600_Default/optimizer-FP32StateAdamWeightDecay/Mul-op25,Default/optimizer-FP32StateAdamWeightDecay/Mul-op25,kernel_graph0,{20326},20326,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20324,20332,0x12c1ae014e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20192,601_Default/optimizer-FP32StateAdamWeightDecay/Mul-op27,Default/optimizer-FP32StateAdamWeightDecay/Mul-op27,kernel_graph0,{20330},20330,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20326,20351,0x12c1b8c15000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20192,602_Default/learning_rate-CosineWithWarmUpLR/Add-op23,Default/learning_rate-CosineWithWarmUpLR/Add-op23,kernel_graph0,{20339-20349},20349,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20330,20349,0x12c1a3414c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20194,603_Default/learning_rate-CosineWithWarmUpLR/Add-op24,Default/learning_rate-CosineWithWarmUpLR/Add-op24,kernel_graph0,{20334-20347},20347,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20334,20338,0x12c1ae014e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20195,604_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op5,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op5,kernel_graph0,{20336},20336,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20336,20341,0x12c1c3815200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20195,605_Default/learning_rate-CosineWithWarmUpLR/Add-op25,Default/learning_rate-CosineWithWarmUpLR/Add-op25,kernel_graph0,{20339},20339,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20339,20344,0x12c1ae014e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20196,606_Default/optimizer-FP32StateAdamWeightDecay/Div-op5,Default/optimizer-FP32StateAdamWeightDecay/Div-op5,kernel_graph0,{20342},20342,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20342,20347,0x12c1c3815200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20197,607_Default/optimizer-FP32StateAdamWeightDecay/Mul-op29,Default/optimizer-FP32StateAdamWeightDecay/Mul-op29,kernel_graph0,{20345},20345,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20345,20353,0x12c1ae014e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20197,608_Default/optimizer-FP32StateAdamWeightDecay/Sub-op5,Default/optimizer-FP32StateAdamWeightDecay/Sub-op5,kernel_graph0,{20351},20351,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20354,20360,0x12c1a3414c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20202,612_Default/optimizer-FP32StateAdamWeightDecay/Mul-op30,Default/optimizer-FP32StateAdamWeightDecay/Mul-op30,kernel_graph0,{20358},20358,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20356,20364,0x12c1ae014e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20203,613_Default/optimizer-FP32StateAdamWeightDecay/Mul-op32,Default/optimizer-FP32StateAdamWeightDecay/Mul-op32,kernel_graph0,{20362},20362,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20358,20383,0x12c1b8c15000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20203,614_Default/learning_rate-CosineWithWarmUpLR/Add-op26,Default/learning_rate-CosineWithWarmUpLR/Add-op26,kernel_graph0,{20371-20381},20381,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20362,20381,0x12c1a3414c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20206,615_Default/learning_rate-CosineWithWarmUpLR/Add-op27,Default/learning_rate-CosineWithWarmUpLR/Add-op27,kernel_graph0,{20366-20379},20379,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20366,20370,0x12c1ae014e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20207,616_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op6,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op6,kernel_graph0,{20368},20368,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20368,20373,0x12c1c3815200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20207,617_Default/learning_rate-CosineWithWarmUpLR/Add-op28,Default/learning_rate-CosineWithWarmUpLR/Add-op28,kernel_graph0,{20371},20371,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20371,20376,0x12c1ae014e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20208,618_Default/optimizer-FP32StateAdamWeightDecay/Div-op6,Default/optimizer-FP32StateAdamWeightDecay/Div-op6,kernel_graph0,{20374},20374,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20374,20379,0x12c1c3815200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20209,619_Default/optimizer-FP32StateAdamWeightDecay/Mul-op34,Default/optimizer-FP32StateAdamWeightDecay/Mul-op34,kernel_graph0,{20377},20377,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20377,20385,0x12c1ae014e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20209,620_Default/optimizer-FP32StateAdamWeightDecay/Sub-op6,Default/optimizer-FP32StateAdamWeightDecay/Sub-op6,kernel_graph0,{20383},20383,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20386,20392,0x12c1a3414c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20214,624_Default/optimizer-FP32StateAdamWeightDecay/Mul-op35,Default/optimizer-FP32StateAdamWeightDecay/Mul-op35,kernel_graph0,{20390},20390,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20388,20396,0x12c1ae014e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20215,625_Default/optimizer-FP32StateAdamWeightDecay/Mul-op37,Default/optimizer-FP32StateAdamWeightDecay/Mul-op37,kernel_graph0,{20394},20394,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20390,20415,0x12c1b8c15000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20215,626_Default/learning_rate-CosineWithWarmUpLR/Add-op29,Default/learning_rate-CosineWithWarmUpLR/Add-op29,kernel_graph0,{20403-20413},20413,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20394,20413,0x12c1a3414c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20217,627_Default/learning_rate-CosineWithWarmUpLR/Add-op30,Default/learning_rate-CosineWithWarmUpLR/Add-op30,kernel_graph0,{20398-20411},20411,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20398,20402,0x12c1ae014e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20218,628_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op7,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op7,kernel_graph0,{20400},20400,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20400,20405,0x12c208812c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20218,629_Default/learning_rate-CosineWithWarmUpLR/Add-op31,Default/learning_rate-CosineWithWarmUpLR/Add-op31,kernel_graph0,{20403},20403,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20403,20408,0x12c1ae014e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20219,630_Default/optimizer-FP32StateAdamWeightDecay/Div-op7,Default/optimizer-FP32StateAdamWeightDecay/Div-op7,kernel_graph0,{20406},20406,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20406,20411,0x12c208812c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20220,631_Default/optimizer-FP32StateAdamWeightDecay/Mul-op39,Default/optimizer-FP32StateAdamWeightDecay/Mul-op39,kernel_graph0,{20409},20409,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20409,20417,0x12c1ae014e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20220,632_Default/optimizer-FP32StateAdamWeightDecay/Sub-op7,Default/optimizer-FP32StateAdamWeightDecay/Sub-op7,kernel_graph0,{20415},20415,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20418,20424,0x12c1e7936800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20222,636_Default/optimizer-FP32StateAdamWeightDecay/Mul-op40,Default/optimizer-FP32StateAdamWeightDecay/Mul-op40,kernel_graph0,{20422},20422,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20420,20428,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20222,637_Default/optimizer-FP32StateAdamWeightDecay/Mul-op42,Default/optimizer-FP32StateAdamWeightDecay/Mul-op42,kernel_graph0,{20426},20426,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20422,20447,0x12c1f5c12600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20223,638_Default/learning_rate-CosineWithWarmUpLR/Add-op32,Default/learning_rate-CosineWithWarmUpLR/Add-op32,kernel_graph0,{20435-20445},20445,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20426,20445,0x12c1e7936800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20223,639_Default/learning_rate-CosineWithWarmUpLR/Add-op33,Default/learning_rate-CosineWithWarmUpLR/Add-op33,kernel_graph0,{20430-20443},20443,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20430,20434,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20224,640_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op8,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op8,kernel_graph0,{20432},20432,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20432,20437,0x12c23a20fa00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20224,641_Default/learning_rate-CosineWithWarmUpLR/Add-op34,Default/learning_rate-CosineWithWarmUpLR/Add-op34,kernel_graph0,{20435},20435,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20435,20440,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20224,642_Default/optimizer-FP32StateAdamWeightDecay/Div-op8,Default/optimizer-FP32StateAdamWeightDecay/Div-op8,kernel_graph0,{20438},20438,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20438,20443,0x12c23a20fa00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20226,643_Default/optimizer-FP32StateAdamWeightDecay/Mul-op44,Default/optimizer-FP32StateAdamWeightDecay/Mul-op44,kernel_graph0,{20441},20441,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20441,20449,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20227,644_Default/optimizer-FP32StateAdamWeightDecay/Sub-op8,Default/optimizer-FP32StateAdamWeightDecay/Sub-op8,kernel_graph0,{20447},20447,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20450,20456,0x12c1e7936800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20229,648_Default/optimizer-FP32StateAdamWeightDecay/Mul-op45,Default/optimizer-FP32StateAdamWeightDecay/Mul-op45,kernel_graph0,{20454},20454,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20452,20460,0x12c23a20fa00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20230,649_Default/optimizer-FP32StateAdamWeightDecay/Mul-op47,Default/optimizer-FP32StateAdamWeightDecay/Mul-op47,kernel_graph0,{20458},20458,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20454,20479,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20231,650_Default/learning_rate-CosineWithWarmUpLR/Add-op35,Default/learning_rate-CosineWithWarmUpLR/Add-op35,kernel_graph0,{20467-20477},20477,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20458,20477,0x12c1f5c12600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20231,651_Default/learning_rate-CosineWithWarmUpLR/Add-op36,Default/learning_rate-CosineWithWarmUpLR/Add-op36,kernel_graph0,{20462-20475},20475,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20462,20466,0x12c1e7936800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20234,652_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op9,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op9,kernel_graph0,{20464},20464,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20464,20469,0x12c23a20fa00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20235,653_Default/learning_rate-CosineWithWarmUpLR/Add-op37,Default/learning_rate-CosineWithWarmUpLR/Add-op37,kernel_graph0,{20467},20467,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20467,20472,0x12c1e7936800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20235,654_Default/optimizer-FP32StateAdamWeightDecay/Div-op9,Default/optimizer-FP32StateAdamWeightDecay/Div-op9,kernel_graph0,{20470},20470,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20470,20475,0x12c23a20fa00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20238,655_Default/optimizer-FP32StateAdamWeightDecay/Mul-op49,Default/optimizer-FP32StateAdamWeightDecay/Mul-op49,kernel_graph0,{20473},20473,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20473,20481,0x12c1e7936800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20239,656_Default/optimizer-FP32StateAdamWeightDecay/Sub-op9,Default/optimizer-FP32StateAdamWeightDecay/Sub-op9,kernel_graph0,{20479},20479,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20482,20488,0x12c1e7936800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20241,660_Default/optimizer-FP32StateAdamWeightDecay/Mul-op50,Default/optimizer-FP32StateAdamWeightDecay/Mul-op50,kernel_graph0,{20486},20486,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20484,20492,0x12c23a20fa00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20243,661_Default/optimizer-FP32StateAdamWeightDecay/Mul-op52,Default/optimizer-FP32StateAdamWeightDecay/Mul-op52,kernel_graph0,{20490},20490,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20486,20511,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20244,662_Default/learning_rate-CosineWithWarmUpLR/Add-op38,Default/learning_rate-CosineWithWarmUpLR/Add-op38,kernel_graph0,{20499-20509},20509,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20490,20509,0x12c1f5c12600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20244,663_Default/learning_rate-CosineWithWarmUpLR/Add-op39,Default/learning_rate-CosineWithWarmUpLR/Add-op39,kernel_graph0,{20494-20507},20507,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20494,20498,0x12c23a20fa00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20246,664_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op10,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op10,kernel_graph0,{20496},20496,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20496,20501,0x12c1e3936600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20247,665_Default/learning_rate-CosineWithWarmUpLR/Add-op40,Default/learning_rate-CosineWithWarmUpLR/Add-op40,kernel_graph0,{20499},20499,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20499,20504,0x12c23a20fa00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20248,666_Default/optimizer-FP32StateAdamWeightDecay/Div-op10,Default/optimizer-FP32StateAdamWeightDecay/Div-op10,kernel_graph0,{20502},20502,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20502,20507,0x12c1e3936600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20249,667_Default/optimizer-FP32StateAdamWeightDecay/Mul-op54,Default/optimizer-FP32StateAdamWeightDecay/Mul-op54,kernel_graph0,{20505},20505,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20505,20513,0x12c23a20fa00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20250,668_Default/optimizer-FP32StateAdamWeightDecay/Sub-op10,Default/optimizer-FP32StateAdamWeightDecay/Sub-op10,kernel_graph0,{20511},20511,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20514,20520,0x12c23a20fa00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20252,672_Default/optimizer-FP32StateAdamWeightDecay/Mul-op55,Default/optimizer-FP32StateAdamWeightDecay/Mul-op55,kernel_graph0,{20518},20518,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20516,20524,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20253,673_Default/optimizer-FP32StateAdamWeightDecay/Mul-op57,Default/optimizer-FP32StateAdamWeightDecay/Mul-op57,kernel_graph0,{20522},20522,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20518,20543,0x12c1f5c12600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20254,674_Default/learning_rate-CosineWithWarmUpLR/Add-op41,Default/learning_rate-CosineWithWarmUpLR/Add-op41,kernel_graph0,{20531-20541},20541,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20522,20541,0x12c23a20fa00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20254,675_Default/learning_rate-CosineWithWarmUpLR/Add-op42,Default/learning_rate-CosineWithWarmUpLR/Add-op42,kernel_graph0,{20526-20539},20539,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20526,20530,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20254,676_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op11,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op11,kernel_graph0,{20528},20528,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20528,20533,0x12c204812a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20255,677_Default/learning_rate-CosineWithWarmUpLR/Add-op43,Default/learning_rate-CosineWithWarmUpLR/Add-op43,kernel_graph0,{20531},20531,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20531,20536,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20256,678_Default/optimizer-FP32StateAdamWeightDecay/Div-op11,Default/optimizer-FP32StateAdamWeightDecay/Div-op11,kernel_graph0,{20534},20534,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20534,20539,0x12c204812a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20256,679_Default/optimizer-FP32StateAdamWeightDecay/Mul-op59,Default/optimizer-FP32StateAdamWeightDecay/Mul-op59,kernel_graph0,{20537},20537,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20537,20545,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20256,680_Default/optimizer-FP32StateAdamWeightDecay/Sub-op11,Default/optimizer-FP32StateAdamWeightDecay/Sub-op11,kernel_graph0,{20543},20543,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20546,20552,0x12c204812a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20259,684_Default/optimizer-FP32StateAdamWeightDecay/Mul-op60,Default/optimizer-FP32StateAdamWeightDecay/Mul-op60,kernel_graph0,{20550},20550,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20548,20556,0x12c19f414a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20260,685_Default/optimizer-FP32StateAdamWeightDecay/Mul-op62,Default/optimizer-FP32StateAdamWeightDecay/Mul-op62,kernel_graph0,{20554},20554,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20550,20575,0x12c1aa014c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20261,686_Default/learning_rate-CosineWithWarmUpLR/Add-op44,Default/learning_rate-CosineWithWarmUpLR/Add-op44,kernel_graph0,{20563-20573},20573,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20554,20573,0x12c204812a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20261,687_Default/learning_rate-CosineWithWarmUpLR/Add-op45,Default/learning_rate-CosineWithWarmUpLR/Add-op45,kernel_graph0,{20558-20571},20571,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20558,20562,0x12c19f414a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20262,688_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op12,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op12,kernel_graph0,{20560},20560,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20560,20565,0x12c22361f800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20263,689_Default/learning_rate-CosineWithWarmUpLR/Add-op46,Default/learning_rate-CosineWithWarmUpLR/Add-op46,kernel_graph0,{20563},20563,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20563,20568,0x12c19f414a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20265,690_Default/optimizer-FP32StateAdamWeightDecay/Div-op12,Default/optimizer-FP32StateAdamWeightDecay/Div-op12,kernel_graph0,{20566},20566,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20566,20571,0x12c22361f800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20266,691_Default/optimizer-FP32StateAdamWeightDecay/Mul-op64,Default/optimizer-FP32StateAdamWeightDecay/Mul-op64,kernel_graph0,{20569},20569,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20569,20577,0x12c19f414a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20267,692_Default/optimizer-FP32StateAdamWeightDecay/Sub-op12,Default/optimizer-FP32StateAdamWeightDecay/Sub-op12,kernel_graph0,{20575},20575,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20578,20584,0x12c22361f800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20272,696_Default/optimizer-FP32StateAdamWeightDecay/Mul-op65,Default/optimizer-FP32StateAdamWeightDecay/Mul-op65,kernel_graph0,{20582},20582,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20580,20588,0x12c204812a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20273,697_Default/optimizer-FP32StateAdamWeightDecay/Mul-op67,Default/optimizer-FP32StateAdamWeightDecay/Mul-op67,kernel_graph0,{20586},20586,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20582,20607,0x12c19f414a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20273,698_Default/learning_rate-CosineWithWarmUpLR/Add-op47,Default/learning_rate-CosineWithWarmUpLR/Add-op47,kernel_graph0,{20595-20605},20605,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20586,20605,0x12c22361f800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20275,699_Default/learning_rate-CosineWithWarmUpLR/Add-op48,Default/learning_rate-CosineWithWarmUpLR/Add-op48,kernel_graph0,{20590-20603},20603,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20590,20594,0x12c22e223c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20276,700_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op13,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op13,kernel_graph0,{20592},20592,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20592,20597,0x12c204812a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20276,701_Default/learning_rate-CosineWithWarmUpLR/Add-op49,Default/learning_rate-CosineWithWarmUpLR/Add-op49,kernel_graph0,{20595},20595,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20595,20600,0x12c22e223c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20278,702_Default/optimizer-FP32StateAdamWeightDecay/Div-op13,Default/optimizer-FP32StateAdamWeightDecay/Div-op13,kernel_graph0,{20598},20598,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20598,20603,0x12c204812a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20279,703_Default/optimizer-FP32StateAdamWeightDecay/Mul-op69,Default/optimizer-FP32StateAdamWeightDecay/Mul-op69,kernel_graph0,{20601},20601,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20601,20609,0x12c22e223c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20279,704_Default/optimizer-FP32StateAdamWeightDecay/Sub-op13,Default/optimizer-FP32StateAdamWeightDecay/Sub-op13,kernel_graph0,{20607},20607,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20610,20616,0x12c22361f800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20283,708_Default/optimizer-FP32StateAdamWeightDecay/Mul-op70,Default/optimizer-FP32StateAdamWeightDecay/Mul-op70,kernel_graph0,{20614},20614,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20612,20620,0x12c22e223c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20284,709_Default/optimizer-FP32StateAdamWeightDecay/Mul-op72,Default/optimizer-FP32StateAdamWeightDecay/Mul-op72,kernel_graph0,{20618},20618,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20614,20639,0x12c204812a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20284,710_Default/learning_rate-CosineWithWarmUpLR/Add-op50,Default/learning_rate-CosineWithWarmUpLR/Add-op50,kernel_graph0,{20627-20637},20637,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20618,20637,0x12c22361f800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20284,711_Default/learning_rate-CosineWithWarmUpLR/Add-op51,Default/learning_rate-CosineWithWarmUpLR/Add-op51,kernel_graph0,{20622-20635},20635,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20622,20626,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20285,712_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op14,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op14,kernel_graph0,{20624},20624,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20624,20629,0x12c22e223c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20286,713_Default/learning_rate-CosineWithWarmUpLR/Add-op52,Default/learning_rate-CosineWithWarmUpLR/Add-op52,kernel_graph0,{20627},20627,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20627,20632,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20286,714_Default/optimizer-FP32StateAdamWeightDecay/Div-op14,Default/optimizer-FP32StateAdamWeightDecay/Div-op14,kernel_graph0,{20630},20630,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20630,20635,0x12c22e223c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20286,715_Default/optimizer-FP32StateAdamWeightDecay/Mul-op74,Default/optimizer-FP32StateAdamWeightDecay/Mul-op74,kernel_graph0,{20633},20633,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20633,20641,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20287,716_Default/optimizer-FP32StateAdamWeightDecay/Sub-op14,Default/optimizer-FP32StateAdamWeightDecay/Sub-op14,kernel_graph0,{20639},20639,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20642,20648,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20290,720_Default/optimizer-FP32StateAdamWeightDecay/Mul-op75,Default/optimizer-FP32StateAdamWeightDecay/Mul-op75,kernel_graph0,{20646},20646,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20644,20652,0x12c19f414a00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20291,721_Default/optimizer-FP32StateAdamWeightDecay/Mul-op77,Default/optimizer-FP32StateAdamWeightDecay/Mul-op77,kernel_graph0,{20650},20650,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20646,20671,0x12c1be814c00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20291,722_Default/learning_rate-CosineWithWarmUpLR/Add-op53,Default/learning_rate-CosineWithWarmUpLR/Add-op53,kernel_graph0,{20659-20669},20669,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20650,20669,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20292,723_Default/learning_rate-CosineWithWarmUpLR/Add-op54,Default/learning_rate-CosineWithWarmUpLR/Add-op54,kernel_graph0,{20654-20667},20667,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20654,20658,0x12c180014800,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20293,724_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op15,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op15,kernel_graph0,{20656},20656,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20656,20661,0x12c19f414a00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20294,725_Default/learning_rate-CosineWithWarmUpLR/Add-op55,Default/learning_rate-CosineWithWarmUpLR/Add-op55,kernel_graph0,{20659},20659,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20659,20664,0x12c180014800,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20294,726_Default/optimizer-FP32StateAdamWeightDecay/Div-op15,Default/optimizer-FP32StateAdamWeightDecay/Div-op15,kernel_graph0,{20662},20662,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20662,20667,0x12c19f414a00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20295,727_Default/optimizer-FP32StateAdamWeightDecay/Mul-op79,Default/optimizer-FP32StateAdamWeightDecay/Mul-op79,kernel_graph0,{20665},20665,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20665,20673,0x12c180014800,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20297,728_Default/optimizer-FP32StateAdamWeightDecay/Sub-op15,Default/optimizer-FP32StateAdamWeightDecay/Sub-op15,kernel_graph0,{20671},20671,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20674,20680,0x12c22361f800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20302,732_Default/optimizer-FP32StateAdamWeightDecay/Mul-op80,Default/optimizer-FP32StateAdamWeightDecay/Mul-op80,kernel_graph0,{20678},20678,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20676,20684,0x12c223623a00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20303,733_Default/optimizer-FP32StateAdamWeightDecay/Mul-op82,Default/optimizer-FP32StateAdamWeightDecay/Mul-op82,kernel_graph0,{20682},20682,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20678,20703,0x12c223627c00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20304,734_Default/learning_rate-CosineWithWarmUpLR/Add-op56,Default/learning_rate-CosineWithWarmUpLR/Add-op56,kernel_graph0,{20691-20701},20701,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20682,20701,0x12c22361b600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20305,735_Default/learning_rate-CosineWithWarmUpLR/Add-op57,Default/learning_rate-CosineWithWarmUpLR/Add-op57,kernel_graph0,{20686-20699},20699,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20686,20690,0x12c22361f800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20305,736_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op16,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op16,kernel_graph0,{20688},20688,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20688,20693,0x12c223623a00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20307,737_Default/learning_rate-CosineWithWarmUpLR/Add-op58,Default/learning_rate-CosineWithWarmUpLR/Add-op58,kernel_graph0,{20691},20691,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20691,20696,0x12c22361f800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20308,738_Default/optimizer-FP32StateAdamWeightDecay/Div-op16,Default/optimizer-FP32StateAdamWeightDecay/Div-op16,kernel_graph0,{20694},20694,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20694,20699,0x12c223623a00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20308,739_Default/optimizer-FP32StateAdamWeightDecay/Mul-op84,Default/optimizer-FP32StateAdamWeightDecay/Mul-op84,kernel_graph0,{20697},20697,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20697,20705,0x12c22361f800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20310,740_Default/optimizer-FP32StateAdamWeightDecay/Sub-op16,Default/optimizer-FP32StateAdamWeightDecay/Sub-op16,kernel_graph0,{20703},20703,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20706,20712,0x12c22361b600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20314,744_Default/optimizer-FP32StateAdamWeightDecay/Mul-op85,Default/optimizer-FP32StateAdamWeightDecay/Mul-op85,kernel_graph0,{20710},20710,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20708,20716,0x12c22361f800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20314,745_Default/optimizer-FP32StateAdamWeightDecay/Mul-op87,Default/optimizer-FP32StateAdamWeightDecay/Mul-op87,kernel_graph0,{20714},20714,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20710,20735,0x12c223623a00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20315,746_Default/learning_rate-CosineWithWarmUpLR/Add-op59,Default/learning_rate-CosineWithWarmUpLR/Add-op59,kernel_graph0,{20723-20733},20733,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20714,20733,0x12c223617400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20316,747_Default/learning_rate-CosineWithWarmUpLR/Add-op60,Default/learning_rate-CosineWithWarmUpLR/Add-op60,kernel_graph0,{20718-20731},20731,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20718,20722,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20316,748_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op17,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op17,kernel_graph0,{20720},20720,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20720,20725,0x12c22361b600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20316,749_Default/learning_rate-CosineWithWarmUpLR/Add-op61,Default/learning_rate-CosineWithWarmUpLR/Add-op61,kernel_graph0,{20723},20723,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20723,20728,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20317,750_Default/optimizer-FP32StateAdamWeightDecay/Div-op17,Default/optimizer-FP32StateAdamWeightDecay/Div-op17,kernel_graph0,{20726},20726,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20726,20731,0x12c22361b600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20318,751_Default/optimizer-FP32StateAdamWeightDecay/Mul-op89,Default/optimizer-FP32StateAdamWeightDecay/Mul-op89,kernel_graph0,{20729},20729,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20729,20737,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20318,752_Default/optimizer-FP32StateAdamWeightDecay/Sub-op17,Default/optimizer-FP32StateAdamWeightDecay/Sub-op17,kernel_graph0,{20735},20735,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20738,20744,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20320,756_Default/optimizer-FP32StateAdamWeightDecay/Mul-op90,Default/optimizer-FP32StateAdamWeightDecay/Mul-op90,kernel_graph0,{20742},20742,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20740,20748,0x12c223617400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20320,757_Default/optimizer-FP32StateAdamWeightDecay/Mul-op92,Default/optimizer-FP32StateAdamWeightDecay/Mul-op92,kernel_graph0,{20746},20746,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20742,20767,0x12c22361b600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20322,758_Default/learning_rate-CosineWithWarmUpLR/Add-op62,Default/learning_rate-CosineWithWarmUpLR/Add-op62,kernel_graph0,{20755-20765},20765,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20746,20765,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20322,759_Default/learning_rate-CosineWithWarmUpLR/Add-op63,Default/learning_rate-CosineWithWarmUpLR/Add-op63,kernel_graph0,{20750-20763},20763,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20750,20754,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20323,760_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op18,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op18,kernel_graph0,{20752},20752,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20752,20757,0x12c22361f800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20323,761_Default/learning_rate-CosineWithWarmUpLR/Add-op64,Default/learning_rate-CosineWithWarmUpLR/Add-op64,kernel_graph0,{20755},20755,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20755,20760,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20324,762_Default/optimizer-FP32StateAdamWeightDecay/Div-op18,Default/optimizer-FP32StateAdamWeightDecay/Div-op18,kernel_graph0,{20758},20758,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20758,20763,0x12c22361f800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20325,763_Default/optimizer-FP32StateAdamWeightDecay/Mul-op94,Default/optimizer-FP32StateAdamWeightDecay/Mul-op94,kernel_graph0,{20761},20761,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20761,20769,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20325,764_Default/optimizer-FP32StateAdamWeightDecay/Sub-op18,Default/optimizer-FP32StateAdamWeightDecay/Sub-op18,kernel_graph0,{20767},20767,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20770,20776,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20330,768_Default/optimizer-FP32StateAdamWeightDecay/Mul-op97,Default/optimizer-FP32StateAdamWeightDecay/Mul-op97,kernel_graph0,{20774},20774,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20772,20780,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20331,769_Default/optimizer-FP32StateAdamWeightDecay/Mul-op95,Default/optimizer-FP32StateAdamWeightDecay/Mul-op95,kernel_graph0,{20778},20778,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20774,20797,0x12c1f1c12400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20333,770_Default/learning_rate-CosineWithWarmUpLR/Add-op66,Default/learning_rate-CosineWithWarmUpLR/Add-op66,kernel_graph0,{20782-20795},20795,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20778,20799,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20334,771_Default/learning_rate-CosineWithWarmUpLR/Add-op65,Default/learning_rate-CosineWithWarmUpLR/Add-op65,kernel_graph0,{20787-20797},20797,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20782,20786,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20335,772_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op19,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op19,kernel_graph0,{20784},20784,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20784,20789,0x12c1f1c16600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20336,773_Default/learning_rate-CosineWithWarmUpLR/Add-op67,Default/learning_rate-CosineWithWarmUpLR/Add-op67,kernel_graph0,{20787},20787,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20787,20792,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20336,774_Default/optimizer-FP32StateAdamWeightDecay/Div-op19,Default/optimizer-FP32StateAdamWeightDecay/Div-op19,kernel_graph0,{20790},20790,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20790,20795,0x12c1f1c16600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20337,775_Default/optimizer-FP32StateAdamWeightDecay/Mul-op99,Default/optimizer-FP32StateAdamWeightDecay/Mul-op99,kernel_graph0,{20793},20793,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20793,20801,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20338,776_Default/optimizer-FP32StateAdamWeightDecay/Sub-op19,Default/optimizer-FP32StateAdamWeightDecay/Sub-op19,kernel_graph0,{20799},20799,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20802,20808,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20342,780_Default/optimizer-FP32StateAdamWeightDecay/Mul-op102,Default/optimizer-FP32StateAdamWeightDecay/Mul-op102,kernel_graph0,{20806},20806,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20804,20812,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20343,781_Default/optimizer-FP32StateAdamWeightDecay/Mul-op100,Default/optimizer-FP32StateAdamWeightDecay/Mul-op100,kernel_graph0,{20810},20810,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20806,20834,0x12c180010600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20344,782_Default/learning_rate-CosineWithWarmUpLR/Add-op69,Default/learning_rate-CosineWithWarmUpLR/Add-op69,kernel_graph0,{20814-20832},20832,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20810,20832,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20345,783_Default/learning_rate-CosineWithWarmUpLR/Add-op68,Default/learning_rate-CosineWithWarmUpLR/Add-op68,kernel_graph0,{20819-20830},20830,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20814,20818,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20346,784_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op20,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op20,kernel_graph0,{20816},20816,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20816,20821,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20347,785_Default/learning_rate-CosineWithWarmUpLR/Add-op70,Default/learning_rate-CosineWithWarmUpLR/Add-op70,kernel_graph0,{20819},20819,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20819,20825,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20348,786_Default/optimizer-FP32StateAdamWeightDecay/Div-op20,Default/optimizer-FP32StateAdamWeightDecay/Div-op20,kernel_graph0,{20822},20822,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20822,20828,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20348,787_Default/optimizer-FP32StateAdamWeightDecay/Mul-op104,Default/optimizer-FP32StateAdamWeightDecay/Mul-op104,kernel_graph0,{20826},20826,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20826,20830,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20348,788_Default/optimizer-FP32StateAdamWeightDecay/Sub-op20,Default/optimizer-FP32StateAdamWeightDecay/Sub-op20,kernel_graph0,{20828},20828,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20835,23134,0x12c180008e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20350,792_Default/TensorMove-op2,Default/TensorMove-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20839,23132,0x12c75d35ec00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/output_actor.cc,689,Other,20837,kernel_graph_0_OutputActor,UpdateOutputDeviceAddress,,{},,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0, 20845,21249,0x12c75d35ee00,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/actor_common.cc,1031,Kernel,20844,0_Default/network-LlamaForCausalLM/StridedSlice-op0,inputs0,0_Default/network-LlamaForCausalLM/StridedSlice-op0,{20846-21247},21247,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,1,0, 20846,21828,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20844,0_Default/network-LlamaForCausalLM/StridedSlice-op0,inputs0,0_Default/network-LlamaForCausalLM/StridedSlice-op0,{20848-20850-21825},21825,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20848,20854,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20846,1_Default/network-LlamaForCausalLM/model-LlamaModel/tok_embeddings-LlamaEmbedding/Gather-op0,Default/network-LlamaForCausalLM/model-LlamaModel/tok_embeddings-LlamaEmbedding/Gather-op0,kernel_graph0,{20852},20852,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20850,20860,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,8704,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20847,2_Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/NotEqual-op0,Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/NotEqual-op0,kernel_graph0,{20855-20858},20858,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20852,21806,0x12c18800c600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20847,3_Default/network-LlamaForCausalLM/model-LlamaModel/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/Cast-op0,kernel_graph0,{20864-20993-21804},21804,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20855,21283,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20847,4_Default/network-LlamaForCausalLM/Cast-op1,Default/network-LlamaForCausalLM/Cast-op1,kernel_graph0,{20856-20875-21236-21280},21280,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20858,20889,0x12c180014600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20847,6_Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Cast-op0,kernel_graph0,{20883-20887},20887,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20861,21732,0x12c180018800,0,DefaultEnhancedAscendMemoryPool,524800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20848,7_Default/network-LlamaForCausalLM/model-LlamaModel/freqs_mgr-FreqsMgr/StridedSlice-op0,Default/network-LlamaForCausalLM/model-LlamaModel/freqs_mgr-FreqsMgr/StridedSlice-op0,kernel_graph0,{20862-20926-20949-21079-21094-21467-21488-21709-21730},21730,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20864,20870,0x12c18c00c800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20849,9_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op0,kernel_graph0,{20868},20868,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20866,20874,0x12c19400ca00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20849,10_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{20872},20872,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20867,21813,0x12c180098a00,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20849,10_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{21809},21809,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20868,20871,0x12c1800a0c00,0,DefaultEnhancedAscendMemoryPool,16777728,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20849,10_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20872,21784,0x12c1800a0c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20850,11_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op1,kernel_graph0,{20897-20902-20917-20940-21754-21768-21782},21782,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20875,20883,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20851,12_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/ReduceSum-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/ReduceSum-op0,kernel_graph0,{20881},20881,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20877,20879,0x12c180008800,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,20877,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20878,20880,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,20878,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20881,21294,0x12c180008800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20851,13_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Add-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Add-op0,kernel_graph0,{21224-21291},21291,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20885,20890,0x12c1840a0e00,0,DefaultEnhancedAscendMemoryPool,2097664,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20853,15_Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/ExpandDims-op0,Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/ExpandDims-op0,kernel_graph0,{20887},20887,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20887,20893,0x12c1842a1000,0,DefaultEnhancedAscendMemoryPool,16777728,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20853,16_Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Mul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Mul-op0,kernel_graph0,{20891},20891,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20891,20897,0x12c1852a1200,0,DefaultEnhancedAscendMemoryPool,16777728,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20855,17_Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Sub-op0,Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Sub-op0,kernel_graph0,{20893-20895},20895,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20895,21690,0x12c1840a0e00,0,DefaultEnhancedAscendMemoryPool,8389120,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20856,19_Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Cast-op1,kernel_graph0,{20978-21125-21443-21684},21684,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20899,21778,0x12c1848a1000,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20857,21_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/Cast-op0,kernel_graph0,{20902-21775},21775,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20901,20909,0x12c18c00c800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20858,22_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,kernel_graph0,{20904-20907},20907,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20902,20904,0x12c1868a1200,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20858,22_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20906,20913,0x12c19000ca00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20860,24_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,kernel_graph0,{20911},20911,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20907,20910,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20860,24_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20911,20930,0x12c19400cc00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20861,25_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op2,kernel_graph0,{20926-20928},20928,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20914,21793,0x12c18c00c800,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20862,26_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/Cast-op0,kernel_graph0,{20917-21790},21790,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20916,20924,0x12c18e00ca00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20862,27_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,kernel_graph0,{20919-20922},20922,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20917,20919,0x12c1868a1200,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20862,27_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20921,20933,0x12c19c00ce00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20862,29_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,kernel_graph0,{20931},20931,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20922,20925,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20862,29_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20926,20969,0x12c1a000d000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20863,30_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op2,kernel_graph0,{20967},20967,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20928,20966,0x12c1a800d200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20864,31_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op1,kernel_graph0,{20964},20964,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20931,20953,0x12c18e00ca00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20865,32_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op0,kernel_graph0,{20949-20951},20951,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20934,21735,0x12c1868a1200,0,DefaultEnhancedAscendMemoryPool,524800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20865,33_Default/network-LlamaForCausalLM/model-LlamaModel/freqs_mgr-FreqsMgr/StridedSlice-op1,Default/network-LlamaForCausalLM/model-LlamaModel/freqs_mgr-FreqsMgr/StridedSlice-op1,kernel_graph0,{20935-20954-20964-21084-21099-21469-21490-21711-21733},21733,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20937,21764,0x12c19600cc00,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20869,35_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/Cast-op0,kernel_graph0,{20940-21761},21761,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20939,20947,0x12c19800ce00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20869,36_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,kernel_graph0,{20942-20945},20945,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20940,20942,0x12c186921400,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20869,36_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20944,21688,0x12c1b000d400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20873,38_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,kernel_graph0,{20978-21684},21684,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20945,20948,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20873,38_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20949,20959,0x12c19800ce00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20873,39_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op0,kernel_graph0,{20957},20957,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20951,20956,0x12c1b400d600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20875,40_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op0,kernel_graph0,{20954},20954,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20954,20960,0x12c18e00ca00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20876,41_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op1,kernel_graph0,{20957},20957,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20957,20963,0x12c1b400d600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20876,42_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op0,kernel_graph0,{20961},20961,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20961,21686,0x12c18e00ca00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20876,43_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op1,kernel_graph0,{20978-21684},21684,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20964,20970,0x12c19800ce00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20876,44_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op3,kernel_graph0,{20967},20967,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20967,20973,0x12c1a800d200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20877,45_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op1,kernel_graph0,{20971},20971,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20971,21687,0x12c19800ce00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20878,46_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op3,kernel_graph0,{20978-21684},21684,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20974,21691,0x12c186921400,0,DefaultEnhancedAscendMemoryPool,8389120,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20878,47_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{21684},21684,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20975,21692,0x12c187121600,0,DefaultEnhancedAscendMemoryPool,8389120,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20878,47_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{21684},21684,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20976,21693,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20878,47_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{21684},21684,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20977,21694,0x12c19c00d000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20878,47_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{20982-21684},21684,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20978,20980,0x12c1a000d200,0,DefaultEnhancedAscendMemoryPool,84936192,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20878,47_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20981,21664,0x12c1a000d200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20880,48_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,kernel_graph0,{20984-20989-21662},21662,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20982,20984,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20880,48_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20986,21672,0x12c19200cc00,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20882,50_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/Cast-op0,kernel_graph0,{20989-21670},21670,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20988,20995,0x12c1a400d400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20882,51_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,kernel_graph0,{20991-20993},20993,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20989,20991,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20882,51_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20993,21644,0x12c1a800d600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20884,53_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/Add-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/Add-op0,kernel_graph0,{20996-21034-21642},21642,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20996,21002,0x12c1b400d600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20885,54_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op0,kernel_graph0,{21000},21000,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20998,21006,0x12c1bc00d800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20886,55_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{21004},21004,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 20999,21651,0x12c187921800,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20886,55_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{21647},21647,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21000,21003,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,16777728,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20886,55_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21004,21623,0x12c1a400d400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20887,56_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op1,kernel_graph0,{21006-21011-21018-21607-21621},21621,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21008,21617,0x12c1b400d600,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20888,58_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/Cast-op0,kernel_graph0,{21011-21614},21614,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21010,21599,0x12c1b960d800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20891,59_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,kernel_graph0,{21013-21024-21597},21597,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21011,21013,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20891,59_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21015,21632,0x12c1c420da00,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20893,61_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/Cast-op0,kernel_graph0,{21018-21629},21629,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21017,21605,0x12c1c980dc00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20894,62_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,kernel_graph0,{21021-21602},21602,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21018,21020,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20894,62_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21021,21595,0x12c1d440de00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20895,63_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/SiLU-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/SiLU-op0,kernel_graph0,{21022-21024-21593},21593,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21024,21582,0x12c1df00e000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20896,65_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/Mul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/Mul-op0,kernel_graph0,{21025-21030-21580},21580,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21027,21590,0x12c1e9c0e200,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20898,67_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/Cast-op0,kernel_graph0,{21030-21588},21588,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21029,21036,0x12c1ef20e400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20899,68_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,kernel_graph0,{21032-21034},21034,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21030,21032,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20899,68_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21034,21562,0x12c1f320e600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20901,70_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/Add-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/Add-op1,kernel_graph0,{21037-21140-21560},21560,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21037,21043,0x12c1f720e800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20902,71_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op0,kernel_graph0,{21041},21041,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21039,21047,0x12c1ff20ea00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20903,72_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{21045},21045,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21040,21569,0x12c187929a00,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20903,72_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{21565},21565,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21041,21044,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,16777728,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20903,72_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21045,21540,0x12c1ef20e400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20904,73_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op1,kernel_graph0,{21047-21052-21067-21112-21510-21524-21538},21538,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21049,21534,0x12c1ac00d800,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20907,75_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/Cast-op0,kernel_graph0,{21052-21531},21531,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21051,21059,0x12c1f720e800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20908,76_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,kernel_graph0,{21054-21057},21057,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21052,21054,0x12c1ae00da00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20908,76_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wk-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21056,21063,0x12c1fb20ea00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20911,78_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,kernel_graph0,{21061},21061,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21057,21060,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20911,78_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21061,21083,0x12c1ff20ec00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20912,79_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op2,kernel_graph0,{21079-21081},21081,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21064,21549,0x12c1f720e800,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20912,80_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/Cast-op0,kernel_graph0,{21067-21546},21546,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21066,21074,0x12c1f920ea00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20914,81_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,kernel_graph0,{21069-21072},21072,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21067,21069,0x12c1ae00da00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20914,81_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wq-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21071,21078,0x12c20720ee00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20915,83_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,kernel_graph0,{21076},21076,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21072,21075,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20915,83_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21076,21098,0x12c20b20f000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20917,84_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op0,kernel_graph0,{21094-21096},21096,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21079,21089,0x12c21320f200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20918,85_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op2,kernel_graph0,{21087},21087,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21081,21086,0x12c21b20f400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20918,86_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op1,kernel_graph0,{21084},21084,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21084,21090,0x12c1f920ea00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20919,87_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op3,kernel_graph0,{21087},21087,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21087,21093,0x12c20120ec00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20920,88_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op1,kernel_graph0,{21091},21091,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21091,21446,0x12c1f920ea00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20922,89_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op3,kernel_graph0,{21125-21443},21443,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21094,21104,0x12c1fd20ec00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20923,90_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op0,kernel_graph0,{21102},21102,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21096,21101,0x12c21320f200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20925,91_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/BatchMatMul-op0,kernel_graph0,{21099},21099,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21099,21105,0x12c20520ee00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20926,92_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Mul-op1,kernel_graph0,{21102},21102,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21102,21108,0x12c20d20f000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20927,93_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Add-op0,kernel_graph0,{21106},21106,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21106,21445,0x12c1fd20ec00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20928,94_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Cast-op1,kernel_graph0,{21125-21443},21443,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21109,21520,0x12c20120ee00,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20928,95_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/Cast-op0,kernel_graph0,{21112-21517},21517,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21111,21119,0x12c20320f000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20929,96_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,kernel_graph0,{21114-21117},21117,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21112,21114,0x12c1ae00da00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20929,96_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wv-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21116,21447,0x12c20720f200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20931,98_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,kernel_graph0,{21125-21443},21443,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21117,21120,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20931,98_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21121,21449,0x12c1ae00da00,0,DefaultEnhancedAscendMemoryPool,8389120,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20932,99_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{21443},21443,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21122,21450,0x12c1ae80dc00,0,DefaultEnhancedAscendMemoryPool,8389120,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20932,99_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{21443},21443,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21123,21451,0x12c180008a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20932,99_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{21443},21443,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21124,21452,0x12c20320f000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20932,99_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{21129-21443},21443,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21125,21127,0x12c20b20f400,0,DefaultEnhancedAscendMemoryPool,84936192,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20932,99_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/FlashAttentionScore-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21128,21423,0x12c20b20f400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20932,100_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,kernel_graph0,{21131-21136-21421},21421,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21129,21131,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20932,100_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/Transpose-op3,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21133,21431,0x12c20f20f600,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20935,102_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/Cast-op0,kernel_graph0,{21136-21429},21429,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21135,21142,0x12c21120f800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20935,103_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,kernel_graph0,{21138-21140},21140,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21136,21138,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20935,103_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/wo-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21140,21398,0x12c21520fa00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20935,105_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/Add-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/Add-op0,kernel_graph0,{21143-21181-21396},21396,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21143,21149,0x12c21920fc00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20935,106_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op0,kernel_graph0,{21147},21147,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21145,21153,0x12c22120fe00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20936,107_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{21151},21151,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21146,21405,0x12c187931c00,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20936,107_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{21401},21401,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21147,21150,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,16777728,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20936,107_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21151,21377,0x12c21120f800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20937,108_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op1,kernel_graph0,{21153-21158-21165-21361-21375},21375,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21155,21371,0x12c21920fc00,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20938,110_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/Cast-op0,kernel_graph0,{21158-21368},21368,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21157,21353,0x12c21e80fe00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20940,111_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,kernel_graph0,{21160-21171-21351},21351,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21158,21160,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20940,111_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w3-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21162,21386,0x12c229410000,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20942,113_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/Cast-op0,kernel_graph0,{21165-21383},21383,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21164,21359,0x12c22ea10200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20943,114_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,kernel_graph0,{21168-21356},21356,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21165,21167,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20943,114_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21168,21349,0x12c239610400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20945,115_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/SiLU-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/SiLU-op0,kernel_graph0,{21169-21171-21347},21347,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21171,21336,0x12c244210600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20946,117_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/Mul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/Mul-op0,kernel_graph0,{21172-21177-21334},21334,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21174,21344,0x12c24ee10800,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20950,119_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/Cast-op0,kernel_graph0,{21177-21342},21342,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21176,21183,0x12c254410a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20950,120_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,kernel_graph0,{21179-21181},21181,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21177,21179,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20950,120_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w2-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21181,21315,0x12c258410c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20952,122_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/Add-op1,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/Add-op1,kernel_graph0,{21184-21313},21313,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21184,21190,0x12c25c410e00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20952,123_Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Cast-op0,Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Cast-op0,kernel_graph0,{21188},21188,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21186,21194,0x12c264411000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20954,124_Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{21192},21192,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21187,21322,0x12c187939e00,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20954,124_Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{21318},21318,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21188,21191,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,16777728,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20954,124_Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/RmsNorm-op0,Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/RmsNorm-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21192,21298,0x12c254410a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20955,125_Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Cast-op1,Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Cast-op1,kernel_graph0,{21194-21199-21296},21296,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21196,21307,0x12c25c410e00,0,DefaultEnhancedAscendMemoryPool,262144512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20994,127_Default/network-LlamaForCausalLM/lm_head-Linear/Cast-op0,Default/network-LlamaForCausalLM/lm_head-Linear/Cast-op0,kernel_graph0,{21199-21304},21304,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21198,21204,0x12c26be11000,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20996,128_Default/network-LlamaForCausalLM/lm_head-Linear/MatMul-op0,Default/network-LlamaForCausalLM/lm_head-Linear/MatMul-op0,kernel_graph0,{21202},21202,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21199,21201,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20996,128_Default/network-LlamaForCausalLM/lm_head-Linear/MatMul-op0,Default/network-LlamaForCausalLM/lm_head-Linear/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21202,21211,0x12c28b211200,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20997,129_Default/network-LlamaForCausalLM/Cast-op0,Default/network-LlamaForCausalLM/Cast-op0,kernel_graph0,{21206-21209},21209,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21205,21212,0x12c187942000,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20997,130_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/ReduceMax-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/ReduceMax-op0,kernel_graph0,{21209},21209,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21206,21208,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,20997,130_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/ReduceMax-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/ReduceMax-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21209,21245,0x12c2c9a11400,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21000,131_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Sub-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Sub-op0,kernel_graph0,{21215-21243},21243,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21213,21226,0x12c180008c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21001,132_Default/Mul-op0,Default/Mul-op0,kernel_graph0,{21224},21224,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21215,21223,0x12c26be11000,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21003,133_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Exp-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Exp-op0,kernel_graph0,{21217},21217,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21217,21242,0x12c187942000,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21004,134_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/ReduceSum-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/ReduceSum-op0,kernel_graph0,{21240},21240,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21219,21221,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,21219,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21220,21222,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,21220,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21224,21230,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21005,135_Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_RealDiv/RealDiv-op0,Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_RealDiv/RealDiv-op0,kernel_graph0,{21228},21228,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21227,21235,0x12c180008c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21006,136_Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_RealDiv/SumExt-op0,Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_RealDiv/SumExt-op0,kernel_graph0,{21231-21233},21233,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21228,21231,0x12c180009600,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21006,136_Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_RealDiv/SumExt-op0,Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_RealDiv/SumExt-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21233,21238,0x12c18794a200,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21008,138_Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_ReduceSum/Tile-op0,Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_ReduceSum/Tile-op0,kernel_graph0,{21236},21236,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21236,21262,0x12c187952400,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21009,139_Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_Mul/Mul-op0,Gradients/Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Grad_Mul/Mul-op0,kernel_graph0,{21238-21259},21259,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21240,21246,0x12c18794a200,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21011,141_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Log-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Log-op0,kernel_graph0,{21243},21243,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21243,21268,0x12c26be11000,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21012,142_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Sub-op1,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/Sub-op1,kernel_graph0,{21254-21266},21266,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21247,21253,0x12c187942000,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21012,143_Default/network-LlamaForCausalLM/StridedSlice-op1,Default/network-LlamaForCausalLM/StridedSlice-op1,kernel_graph0,{21249-21251},21251,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21251,21269,0x12c2aa611200,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21014,145_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/OneHot-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_log_softmax-_LogSoftmax/OneHot-op0,kernel_graph0,{21256-21266},21266,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21254,21258,0x12c2e8e11400,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21015,146_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Exp-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Exp-op0,kernel_graph0,{21256},21256,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21256,21261,0x12c327611600,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21016,147_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Sub-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Sub-op0,kernel_graph0,{21259},21259,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21259,21265,0x12c2e8e11400,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21017,148_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Mul-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Mul-op0,kernel_graph0,{21263},21263,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21263,21306,0x12c327611600,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21018,149_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op0,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op0,kernel_graph0,{21296-21304},21304,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21266,21272,0x12c2e8e11400,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21019,150_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Mul-op1,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Mul-op1,kernel_graph0,{21270},21270,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21270,21279,0x12c26be11000,0,DefaultEnhancedAscendMemoryPool,1048576512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21019,151_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Neg-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/Neg-op0,kernel_graph0,{21273},21273,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21273,21282,0x12c187942000,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21021,152_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/ReduceSum-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/_nllloss-_NLLLoss/ReduceSum-op0,kernel_graph0,{21280},21280,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21275,21277,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,21275,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21276,21278,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,21276,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21280,21290,0x12c18794a200,0,DefaultEnhancedAscendMemoryPool,33280,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21022,153_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Mul-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/Mul-op0,kernel_graph0,{21284},21284,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21284,21293,0x12c180008c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21022,154_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/ReduceSum-op1,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/ReduceSum-op1,kernel_graph0,{21291},21291,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21286,21288,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,21286,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21287,21289,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,1024,16181624832,mindspore/ops/kernel/ascend/acl_ir/acl_allocator.cc,46,WorkSpace,21287,AclWorkspace,AclWorkspace,,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21291,23127,0x12c180009400,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21022,155_Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/RealDiv-op0,Default/network-LlamaForCausalLM/loss-CrossEntropyLoss/RealDiv-op0,kernel_graph0,{23125},23125,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21295,21302,0x12c346a11800,0,DefaultEnhancedAscendMemoryPool,262144512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21023,156_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op29,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op29,kernel_graph0,{21300},21300,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21296,21299,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21023,156_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op29,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op29,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21300,21881,0x12c26be11000,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21024,157_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op29,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op29,kernel_graph0,{21879},21879,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21303,21312,0x12c254410a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21025,158_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op0,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op0,kernel_graph0,{21308-21310},21310,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21304,21308,0x12c346a11800,0,DefaultEnhancedAscendMemoryPool,155190272,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21025,158_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op0,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21310,21320,0x12c25c410e00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21026,160_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op1,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op1,kernel_graph0,{21318},21318,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21313,21321,0x12c28b211200,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21027,161_Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Cast-op2,kernel_graph0,{21318},21318,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21316,21326,0x12c254410a00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21028,162_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op0,kernel_graph0,{21324},21324,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21317,21896,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21028,162_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op0,kernel_graph0,{21894},21894,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21318,21323,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,16778752,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21028,162_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21324,21413,0x12c25c410c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21028,163_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op0,kernel_graph0,{21328-21334-21342-21410},21410,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21327,21332,0x12c180008800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21030,164_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Mul-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Mul-op0,kernel_graph0,{21330},21330,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21330,22096,0x12c180008c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21032,166_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Maximum-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Maximum-op0,kernel_graph0,{22094},22094,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21333,21340,0x12c254410a00,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21033,167_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op27,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op27,kernel_graph0,{21338},21338,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21334,21337,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21033,167_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op27,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op27,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21338,21875,0x12c244210600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21034,168_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op27,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op27,kernel_graph0,{21873},21873,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21341,21354,0x12c260410e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21035,169_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op1,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op1,kernel_graph0,{21345-21347-21351},21351,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21342,21345,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21035,169_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op1,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21347,21370,0x12c24ee10800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21037,171_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op0,kernel_graph0,{21349-21361-21368},21368,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21351,21358,0x12c239610400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21038,173_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op1,kernel_graph0,{21354-21356},21356,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21356,21385,0x12c21e80fe00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21042,175_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/Grad_SiLU/SiLUGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/Grad_SiLU/SiLUGrad-op0,kernel_graph0,{21375-21383},21383,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21360,21366,0x12c260410e00,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21042,176_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op28,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op28,kernel_graph0,{21364},21364,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21361,21363,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21042,176_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op28,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op28,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21364,21878,0x12c22ea10200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21045,177_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op28,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op28,kernel_graph0,{21876},21876,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21367,21391,0x12c239610400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21046,178_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op2,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op2,kernel_graph0,{21372-21389},21389,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21368,21372,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21046,178_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op2,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21374,21381,0x12c21920fc00,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21047,180_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op26,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op26,kernel_graph0,{21379},21379,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21375,21378,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21047,180_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op26,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op26,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21379,21872,0x12c260410e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21048,181_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op26,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op26,kernel_graph0,{21870},21870,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21382,21392,0x12c21120f800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21049,182_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op3,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op3,kernel_graph0,{21387-21389},21389,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21383,21387,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21049,182_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op3,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op3,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21389,21395,0x12c23d610600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21051,184_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/AddN-op0,kernel_graph0,{21393},21393,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21393,21403,0x12c24ee10800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21052,185_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op2,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op2,kernel_graph0,{21401},21401,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21396,21404,0x12c239610400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21053,186_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op2,kernel_graph0,{21401},21401,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21399,21409,0x12c21120f800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21053,187_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op1,kernel_graph0,{21407},21407,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21400,21890,0x12c180010600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21053,187_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op1,kernel_graph0,{21888},21888,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21401,21406,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,16778752,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21053,187_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21407,21412,0x12c239610400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21054,188_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op1,kernel_graph0,{21410},21410,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21410,21577,0x12c23d610600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21055,189_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/AddN-op0,kernel_graph0,{21415-21421-21429-21574},21574,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21414,21419,0x12c180008800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21057,190_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Sub-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Sub-op0,kernel_graph0,{21417},21417,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21417,22119,0x12c180009600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21060,192_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/LessEqual-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/LessEqual-op0,kernel_graph0,{22099-22117},22117,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21420,21427,0x12c241610800,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21061,193_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op25,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op25,kernel_graph0,{21425},21425,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21421,21424,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21061,193_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op25,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op25,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21425,21869,0x12c20b20f400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21062,194_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op25,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op25,kernel_graph0,{21867},21867,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21428,21437,0x12c239610400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21063,195_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op4,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op4,kernel_graph0,{21432-21435},21435,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21429,21432,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21063,195_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op4,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op4,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21434,21448,0x12c24ee10800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21065,197_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op0,kernel_graph0,{21443},21443,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21435,21438,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21065,197_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21439,21460,0x12c239610400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21065,198_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,kernel_graph0,{21458},21458,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21440,21457,0x12c252e10a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21065,198_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,kernel_graph0,{21455},21455,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21441,21464,0x12c256e10c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21065,198_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,kernel_graph0,{21462},21462,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21442,21453,0x12c180008800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21065,198_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21443,21454,0x12c20f20f600,0,DefaultEnhancedAscendMemoryPool,515969024,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21065,198_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21455,21471,0x12c1f920ea00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21067,199_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op5,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op5,kernel_graph0,{21467-21469},21469,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21458,21492,0x12c20320f000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21068,200_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op3,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op3,kernel_graph0,{21488-21490},21490,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21461,21519,0x12c239610400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21069,201_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op3,kernel_graph0,{21465-21510-21517},21517,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21462,21465,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21069,201_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op3,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21467,21477,0x12c24ee10800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21070,203_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op4,kernel_graph0,{21475},21475,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21469,21474,0x12c256e10a00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21072,204_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op5,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op5,kernel_graph0,{21472},21472,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21472,21478,0x12c1f920ea00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21073,205_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op1,kernel_graph0,{21475},21475,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21475,21481,0x12c256e10a00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21075,206_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op1,kernel_graph0,{21479},21479,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21479,21485,0x12c24ee10800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21076,207_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op6,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op6,kernel_graph0,{21483},21483,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21482,21533,0x12c1f920ea00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21077,208_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op2,kernel_graph0,{21486-21524-21531},21531,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21483,21486,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21077,208_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21488,21498,0x12c24ee10800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21079,210_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op2,kernel_graph0,{21496},21496,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21490,21495,0x12c256e10a00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21080,211_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op3,kernel_graph0,{21493},21493,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21493,21499,0x12c20320f000,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21080,212_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op0,kernel_graph0,{21496},21496,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21496,21502,0x12c256e10a00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21081,213_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op0,kernel_graph0,{21500},21500,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21500,21506,0x12c1fd20ec00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21082,214_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op4,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op4,kernel_graph0,{21504},21504,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21503,21548,0x12c20320f000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21082,215_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op1,kernel_graph0,{21507-21538-21546},21546,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21504,21507,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21082,215_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21509,21515,0x12c241610800,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21085,217_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op24,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op24,kernel_graph0,{21513},21513,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21510,21512,0x12c1ae00da00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21085,217_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op24,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op24,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21513,21866,0x12c1fd20ec00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21085,218_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op24,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op24,kernel_graph0,{21864},21864,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21516,21556,0x12c20720f200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21087,219_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op7,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op7,kernel_graph0,{21521-21552},21552,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21517,21521,0x12c1ae00da00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21087,219_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op7,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op7,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21523,21529,0x12c20120ee00,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21088,221_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op23,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op23,kernel_graph0,{21527},21527,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21524,21526,0x12c1ae00da00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21088,221_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op23,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op23,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21527,21863,0x12c239610400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21091,222_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op23,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op23,kernel_graph0,{21861},21861,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21530,21555,0x12c24ee10800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21092,223_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op6,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op6,kernel_graph0,{21535-21552},21552,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21531,21535,0x12c1ae00da00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21092,223_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op6,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op6,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21537,21544,0x12c20120ee00,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21094,225_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op22,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op22,kernel_graph0,{21542},21542,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21538,21541,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21094,225_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op22,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op22,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21542,21860,0x12c1ef20e400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21095,226_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op22,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op22,kernel_graph0,{21858},21858,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21545,21554,0x12c1f920ea00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21095,227_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op5,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op5,kernel_graph0,{21550-21552},21552,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21546,21550,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21095,227_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op5,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op5,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21552,21559,0x12c20120ee00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21097,229_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/AddN-op0,kernel_graph0,{21557},21557,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21557,21567,0x12c24ee10800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21097,230_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op7,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op7,kernel_graph0,{21565},21565,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21560,21568,0x12c256e10a00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21099,231_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/1-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op2,kernel_graph0,{21565},21565,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21563,21573,0x12c1f320e600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21100,232_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op2,kernel_graph0,{21571},21571,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21564,21893,0x12c187931c00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21100,232_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op2,kernel_graph0,{21891},21891,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21565,21570,0x12c25ee10c00,0,DefaultEnhancedAscendMemoryPool,16778752,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21100,232_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21571,21576,0x12c20120ee00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21100,233_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op2,kernel_graph0,{21574},21574,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21574,21659,0x12c20520f000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21102,234_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/AddN-op0,kernel_graph0,{21577-21580-21588-21656},21656,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21579,21586,0x12c23d610600,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21105,236_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op20,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op20,kernel_graph0,{21584},21584,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21580,21583,0x12c242c10800,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21105,236_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op20,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op20,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21584,21854,0x12c1df00e000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21106,237_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op20,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op20,kernel_graph0,{21852},21852,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21587,21600,0x12c24ee10800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21107,238_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op8,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op8,kernel_graph0,{21591-21593-21597},21597,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21588,21591,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21107,238_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op8,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op8,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21593,21616,0x12c20f20f600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21110,240_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op6,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op6,kernel_graph0,{21595-21607-21614},21614,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21597,21604,0x12c1d440de00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21112,242_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op7,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op7,kernel_graph0,{21600-21602},21602,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21602,21631,0x12c1b960d800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21113,244_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/Grad_SiLU/SiLUGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/feed_forward-LlamaFeedForward/w1-Linear/activation-LlamaSiLU/Grad_SiLU/SiLUGrad-op1,kernel_graph0,{21621-21629},21629,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21606,21612,0x12c1e9c0e200,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21114,245_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op21,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op21,kernel_graph0,{21610},21610,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21607,21609,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21114,245_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op21,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op21,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21610,21857,0x12c24ee10800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21115,246_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op21,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op21,kernel_graph0,{21855},21855,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21613,21637,0x12c20120ee00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21117,247_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op9,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op9,kernel_graph0,{21618-21635},21635,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21614,21618,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21117,247_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op9,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op9,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21620,21627,0x12c1b400d600,0,DefaultEnhancedAscendMemoryPool,90178048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21118,249_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op19,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op19,kernel_graph0,{21625},21625,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21621,21624,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21118,249_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op19,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op19,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21625,21851,0x12c1c980dc00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21124,250_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op19,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op19,kernel_graph0,{21849},21849,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21628,21638,0x12c1a400d400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21125,251_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op10,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op10,kernel_graph0,{21633-21635},21635,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21629,21633,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21125,251_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op10,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op10,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21635,21641,0x12c1e9c0e200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21126,253_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/AddN-op0,kernel_graph0,{21639},21639,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21639,21649,0x12c1f320e600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21127,254_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op8,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op8,kernel_graph0,{21647},21647,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21642,21650,0x12c1d440de00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21129,255_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/ffn_norm-LlamaRMSNorm/Cast-op2,kernel_graph0,{21647},21647,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21645,21655,0x12c1a400d400,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21130,256_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op3,kernel_graph0,{21653},21653,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21646,21884,0x12c187929a00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21130,256_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op3,kernel_graph0,{21882},21882,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21647,21652,0x12c19400ce00,0,DefaultEnhancedAscendMemoryPool,16778752,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21130,256_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op3,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21653,21658,0x12c20120ee00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21131,257_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op3,kernel_graph0,{21656},21656,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21656,21821,0x12c1e9c0e200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21131,258_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/AddN-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/AddN-op1,kernel_graph0,{21659-21662-21670-21818},21818,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21661,21668,0x12c259a10a00,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21133,260_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op18,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op18,kernel_graph0,{21666},21666,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21662,21665,0x12c1edc0e400,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21133,260_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op18,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op18,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21666,21848,0x12c25ba10c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21134,261_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op18,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op18,kernel_graph0,{21846},21846,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21669,21678,0x12c23d610600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21135,262_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op11,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op11,kernel_graph0,{21673-21676},21676,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21670,21673,0x12c1edc0e400,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21135,262_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op11,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op11,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21675,21689,0x12c1f320e600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21137,264_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op4,kernel_graph0,{21684},21684,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21676,21679,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21137,264_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op4,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21680,21702,0x12c1f720e800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21137,265_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,kernel_graph0,{21700},21700,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21681,21699,0x12c23d610600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21137,265_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,kernel_graph0,{21697},21697,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21682,21706,0x12c20120ee00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21137,265_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,kernel_graph0,{21704},21704,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21683,21695,0x12c180008800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21137,265_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21684,21696,0x12c20f20f600,0,DefaultEnhancedAscendMemoryPool,515969024,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21137,265_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/flash_attention-FlashAttention/Grad_FlashAttentionScore/FlashAttentionScoreGrad-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21697,21713,0x12c18e00ca00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21138,266_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op11,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op11,kernel_graph0,{21709-21711},21711,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21700,21736,0x12c1d440de00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21139,267_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op9,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op9,kernel_graph0,{21730-21733},21733,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21703,21763,0x12c20520f000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21140,268_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op7,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op7,kernel_graph0,{21707-21754-21761},21761,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21704,21707,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21140,268_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op7,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op7,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21709,21719,0x12c1f320e600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21142,270_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op10,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op10,kernel_graph0,{21717},21717,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21711,21716,0x12c20f20f600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21143,271_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op11,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op11,kernel_graph0,{21714},21714,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21714,21720,0x12c18e00ca00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21144,272_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op3,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op3,kernel_graph0,{21717},21717,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21717,21723,0x12c20f20f600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21145,273_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op1,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op1,kernel_graph0,{21721},21721,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21721,21727,0x12c20120ee00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21147,274_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op12,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op12,kernel_graph0,{21725},21725,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21724,21777,0x12c23d610600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21148,275_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op6,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op6,kernel_graph0,{21728-21768-21775},21775,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21725,21728,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21148,275_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op6,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op6,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21730,21742,0x12c18e00ca00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21151,277_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op8,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op8,kernel_graph0,{21740},21740,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21733,21739,0x12c1f320e600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21152,278_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op9,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/casual_mask-LowerTriangularMaskWithDynamic/Grad_Mul/Mul-op9,kernel_graph0,{21737},21737,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21737,21743,0x12c1d440de00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21152,279_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op2,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/Grad_BatchMatMul/BatchMatMul-op2,kernel_graph0,{21740},21740,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21740,21746,0x12c1f320e600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21153,280_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/apply_rotary_emb-RotaryEmbedding/AddN-op0,kernel_graph0,{21744},21744,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21744,21750,0x12c20120ee00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21154,281_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op10,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op10,kernel_graph0,{21748},21748,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21747,21792,0x12c18e00ca00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21155,282_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op5,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op5,kernel_graph0,{21751-21782-21790},21790,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21748,21751,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,2048,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21155,282_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op5,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention-LLamaAttention/Grad_Transpose/Transpose-op5,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21753,21759,0x12c20920f200,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21157,284_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op17,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op17,kernel_graph0,{21757},21757,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21754,21756,0x12c1edc0e400,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21157,284_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op17,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op17,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21757,21845,0x12c20120ee00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21158,285_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op17,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op17,kernel_graph0,{21843},21843,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21760,21800,0x12c1f320e600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21159,286_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op14,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op14,kernel_graph0,{21765-21796},21796,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21761,21765,0x12c1edc0e400,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21159,286_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op14,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op14,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21767,21773,0x12c259a10a00,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21160,288_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op16,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op16,kernel_graph0,{21771},21771,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21768,21770,0x12c1edc0e400,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21160,288_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op16,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op16,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21771,21842,0x12c1f720e800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21161,289_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op16,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op16,kernel_graph0,{21840},21840,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21774,21799,0x12c20520f000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21162,290_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op13,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op13,kernel_graph0,{21779-21796},21796,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21775,21779,0x12c1edc0e400,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21162,290_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op13,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op13,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21781,21788,0x12c1fb20ea00,0,DefaultEnhancedAscendMemoryPool,33554944,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21165,292_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op15,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op15,kernel_graph0,{21786},21786,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21782,21785,0x12c1edc0e400,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21165,292_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op15,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op15,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21786,21839,0x12c23d610600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21166,293_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op15,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op15,kernel_graph0,{21837},21837,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21789,21798,0x12c1800a0c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21166,294_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op12,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op12,kernel_graph0,{21794-21796},21796,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21790,21794,0x12c1edc0e400,0,DefaultEnhancedAscendMemoryPool,20972544,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21166,294_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op12,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_MatMul/MatMul-op12,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21796,21803,0x12c1d440de00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21169,296_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/AddN-op0,kernel_graph0,{21801},21801,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21801,21811,0x12c20f20f600,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21169,297_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op13,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op13,kernel_graph0,{21809},21809,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21804,21812,0x12c1d440de00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21170,298_Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op2,Default/network-LlamaForCausalLM/model-LlamaModel/layers-CellList/0-LLamaDecodeLayer/attention_norm-LlamaRMSNorm/Cast-op2,kernel_graph0,{21809},21809,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21807,21817,0x12c21720f800,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21171,299_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op4,kernel_graph0,{21815},21815,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21808,21887,0x12c180014800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21171,299_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op4,kernel_graph0,{21885},21885,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21809,21814,0x12c1edc0e400,0,DefaultEnhancedAscendMemoryPool,16778752,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21171,299_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_RmsNorm/RmsNormGrad-op4,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21815,21820,0x12c1f320e600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21172,300_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op4,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/norm_out-LlamaRMSNorm/Grad_Cast/Cast-op4,kernel_graph0,{21818},21818,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21818,21824,0x12c20520f000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21172,301_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/AddN-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/AddN-op0,kernel_graph0,{21822},21822,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21822,21827,0x12c1d440de00,0,DefaultEnhancedAscendMemoryPool,134218240,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21172,302_Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op14,Gradients/Default/network-LlamaForCausalLM/lm_head-Linear/Grad_Cast/Cast-op14,kernel_graph0,{21825},21825,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21825,21836,0x12c20f20f600,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21173,303_Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/tok_embeddings-LlamaEmbedding/Grad_Gather/UnsortedSegmentSum-op0,Gradients/Default/network-LlamaForCausalLM/model-LlamaModel/tok_embeddings-LlamaEmbedding/Grad_Gather/UnsortedSegmentSum-op0,kernel_graph0,{21834},21834,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21829,21833,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21174,304_Default/Mul-op1,Default/Mul-op1,kernel_graph0,{21831},21831,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21831,21897,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21175,305_Default/Reciprocal-op0,Default/Reciprocal-op0,kernel_graph0,{21834-21837-21840-21843-21846-21849-21852-21855-21858-21861-21864-21867-21870-21873-21876-21879-21882-21885-21888-21891-21894},21894,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21834,22132,0x12c187935e00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21176,306_Default/Mul-op2,Default/Mul-op2,kernel_graph0,{21898-22058-22130},22130,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21837,22135,0x12c1f320e600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21177,307_Default/Mul-op3,Default/Mul-op3,kernel_graph0,{21906-22058-22133},22133,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21840,22138,0x12c1e9c0e200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21178,308_Default/Mul-op4,Default/Mul-op4,kernel_graph0,{21914-22058-22136},22136,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21843,22141,0x12c1f720e800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21179,309_Default/Mul-op5,Default/Mul-op5,kernel_graph0,{21922-22058-22139},22139,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21846,22144,0x12c23d610600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21180,310_Default/Mul-op6,Default/Mul-op6,kernel_graph0,{21930-22058-22142},22142,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21849,22147,0x12c1d440de00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21181,311_Default/Mul-op7,Default/Mul-op7,kernel_graph0,{21938-22058-22145},22145,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21852,22150,0x12c20f20f600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21182,312_Default/Mul-op8,Default/Mul-op8,kernel_graph0,{21946-22058-22148},22148,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21855,22153,0x12c1df00e000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21182,313_Default/Mul-op9,Default/Mul-op9,kernel_graph0,{21954-22058-22151},22151,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21858,22156,0x12c180018a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21184,314_Default/Mul-op10,Default/Mul-op10,kernel_graph0,{21962-22058-22154},22154,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21861,22159,0x12c1edc0e400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21185,315_Default/Mul-op11,Default/Mul-op11,kernel_graph0,{21970-22058-22157},22157,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21864,22162,0x12c239610400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21185,316_Default/Mul-op12,Default/Mul-op12,kernel_graph0,{21978-22058-22160},22160,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21867,22165,0x12c1fb20ea00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21187,317_Default/Mul-op13,Default/Mul-op13,kernel_graph0,{21986-22058-22163},22163,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21870,22168,0x12c1ff20ec00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21188,318_Default/Mul-op14,Default/Mul-op14,kernel_graph0,{21994-22058-22166},22166,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21873,22171,0x12c219e0f800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21189,319_Default/Mul-op15,Default/Mul-op15,kernel_graph0,{22002-22058-22169},22169,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21876,22174,0x12c241610800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21189,320_Default/Mul-op16,Default/Mul-op16,kernel_graph0,{22010-22058-22172},22172,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21879,22177,0x12c24c210a00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21192,321_Default/Mul-op17,Default/Mul-op17,kernel_graph0,{22018-22058-22175},22175,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21882,22180,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21193,322_Default/Mul-op18,Default/Mul-op18,kernel_graph0,{22026-22058-22178},22178,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21885,22183,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21193,323_Default/Mul-op19,Default/Mul-op19,kernel_graph0,{22034-22058-22181},22181,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21888,22186,0x12c180014800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21194,324_Default/Mul-op20,Default/Mul-op20,kernel_graph0,{22042-22058-22184},22184,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21891,22189,0x12c180010600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21195,325_Default/Mul-op21,Default/Mul-op21,kernel_graph0,{22050-22058-22187},22187,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21894,22192,0x12c1f1c0e600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21196,326_Default/Mul-op22,Default/Mul-op22,kernel_graph0,{22058-22060-22190},22190,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21898,21903,0x12c1a6d36000,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21197,327_Default/clip_grad_norm-ClipGradNorm/Square-op0,Default/clip_grad_norm-ClipGradNorm/Square-op0,kernel_graph0,{21901},21901,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21900,22070,0x12c180008a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21197,328_Default/clip_grad_norm-ClipGradNorm/SumExt-op0,Default/clip_grad_norm-ClipGradNorm/SumExt-op0,kernel_graph0,{21904-22068},22068,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21901,21904,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21197,328_Default/clip_grad_norm-ClipGradNorm/SumExt-op0,Default/clip_grad_norm-ClipGradNorm/SumExt-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21906,21911,0x12c209e0ee00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21200,330_Default/clip_grad_norm-ClipGradNorm/Square-op1,Default/clip_grad_norm-ClipGradNorm/Square-op1,kernel_graph0,{21909},21909,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21908,22071,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21200,331_Default/clip_grad_norm-ClipGradNorm/SumExt-op1,Default/clip_grad_norm-ClipGradNorm/SumExt-op1,kernel_graph0,{21912-22068},22068,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21909,21912,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21200,331_Default/clip_grad_norm-ClipGradNorm/SumExt-op1,Default/clip_grad_norm-ClipGradNorm/SumExt-op1,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21914,21919,0x12c209e0ee00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21203,333_Default/clip_grad_norm-ClipGradNorm/Square-op2,Default/clip_grad_norm-ClipGradNorm/Square-op2,kernel_graph0,{21917},21917,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21916,22072,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21203,334_Default/clip_grad_norm-ClipGradNorm/SumExt-op2,Default/clip_grad_norm-ClipGradNorm/SumExt-op2,kernel_graph0,{21920-22068},22068,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21917,21920,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21203,334_Default/clip_grad_norm-ClipGradNorm/SumExt-op2,Default/clip_grad_norm-ClipGradNorm/SumExt-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21922,21927,0x12c209e0ee00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21207,336_Default/clip_grad_norm-ClipGradNorm/Square-op3,Default/clip_grad_norm-ClipGradNorm/Square-op3,kernel_graph0,{21925},21925,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21924,22073,0x12c180009a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21207,337_Default/clip_grad_norm-ClipGradNorm/SumExt-op3,Default/clip_grad_norm-ClipGradNorm/SumExt-op3,kernel_graph0,{21928-22068},22068,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21925,21928,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21207,337_Default/clip_grad_norm-ClipGradNorm/SumExt-op3,Default/clip_grad_norm-ClipGradNorm/SumExt-op3,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21930,21935,0x12c209e0ee00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21209,339_Default/clip_grad_norm-ClipGradNorm/Square-op4,Default/clip_grad_norm-ClipGradNorm/Square-op4,kernel_graph0,{21933},21933,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21932,22074,0x12c180009c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21210,340_Default/clip_grad_norm-ClipGradNorm/SumExt-op4,Default/clip_grad_norm-ClipGradNorm/SumExt-op4,kernel_graph0,{21936-22068},22068,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21933,21936,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21210,340_Default/clip_grad_norm-ClipGradNorm/SumExt-op4,Default/clip_grad_norm-ClipGradNorm/SumExt-op4,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21938,21943,0x12c224a0fa00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21213,342_Default/clip_grad_norm-ClipGradNorm/Square-op5,Default/clip_grad_norm-ClipGradNorm/Square-op5,kernel_graph0,{21941},21941,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21940,22075,0x12c180009e00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21214,343_Default/clip_grad_norm-ClipGradNorm/SumExt-op5,Default/clip_grad_norm-ClipGradNorm/SumExt-op5,kernel_graph0,{21944-22068},22068,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21941,21944,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21214,343_Default/clip_grad_norm-ClipGradNorm/SumExt-op5,Default/clip_grad_norm-ClipGradNorm/SumExt-op5,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21946,21951,0x12c224a0fa00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21215,345_Default/clip_grad_norm-ClipGradNorm/Square-op6,Default/clip_grad_norm-ClipGradNorm/Square-op6,kernel_graph0,{21949},21949,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21948,22076,0x12c18000a000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21216,346_Default/clip_grad_norm-ClipGradNorm/SumExt-op6,Default/clip_grad_norm-ClipGradNorm/SumExt-op6,kernel_graph0,{21952-22068},22068,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21949,21952,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21216,346_Default/clip_grad_norm-ClipGradNorm/SumExt-op6,Default/clip_grad_norm-ClipGradNorm/SumExt-op6,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21954,21959,0x12c224a0fa00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21217,348_Default/clip_grad_norm-ClipGradNorm/Square-op7,Default/clip_grad_norm-ClipGradNorm/Square-op7,kernel_graph0,{21957},21957,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21956,22077,0x12c18000a200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21218,349_Default/clip_grad_norm-ClipGradNorm/SumExt-op7,Default/clip_grad_norm-ClipGradNorm/SumExt-op7,kernel_graph0,{21960-22068},22068,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21957,21960,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21218,349_Default/clip_grad_norm-ClipGradNorm/SumExt-op7,Default/clip_grad_norm-ClipGradNorm/SumExt-op7,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21962,21967,0x12c209e0ee00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21218,351_Default/clip_grad_norm-ClipGradNorm/Square-op8,Default/clip_grad_norm-ClipGradNorm/Square-op8,kernel_graph0,{21965},21965,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21964,22078,0x12c18000a400,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21218,352_Default/clip_grad_norm-ClipGradNorm/SumExt-op8,Default/clip_grad_norm-ClipGradNorm/SumExt-op8,kernel_graph0,{21968-22068},22068,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21965,21968,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21218,352_Default/clip_grad_norm-ClipGradNorm/SumExt-op8,Default/clip_grad_norm-ClipGradNorm/SumExt-op8,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21970,21975,0x12c209e0ee00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21219,354_Default/clip_grad_norm-ClipGradNorm/Square-op9,Default/clip_grad_norm-ClipGradNorm/Square-op9,kernel_graph0,{21973},21973,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21972,22079,0x12c18000a600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21220,355_Default/clip_grad_norm-ClipGradNorm/SumExt-op9,Default/clip_grad_norm-ClipGradNorm/SumExt-op9,kernel_graph0,{21976-22068},22068,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21973,21976,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21220,355_Default/clip_grad_norm-ClipGradNorm/SumExt-op9,Default/clip_grad_norm-ClipGradNorm/SumExt-op9,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21978,21983,0x12c209e0ee00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21224,357_Default/clip_grad_norm-ClipGradNorm/Square-op10,Default/clip_grad_norm-ClipGradNorm/Square-op10,kernel_graph0,{21981},21981,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21980,22080,0x12c18000a800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21224,358_Default/clip_grad_norm-ClipGradNorm/SumExt-op10,Default/clip_grad_norm-ClipGradNorm/SumExt-op10,kernel_graph0,{21984-22068},22068,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21981,21984,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21224,358_Default/clip_grad_norm-ClipGradNorm/SumExt-op10,Default/clip_grad_norm-ClipGradNorm/SumExt-op10,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21986,21991,0x12c209e0ee00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21225,360_Default/clip_grad_norm-ClipGradNorm/Square-op11,Default/clip_grad_norm-ClipGradNorm/Square-op11,kernel_graph0,{21989},21989,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21988,22081,0x12c18000aa00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21225,361_Default/clip_grad_norm-ClipGradNorm/SumExt-op11,Default/clip_grad_norm-ClipGradNorm/SumExt-op11,kernel_graph0,{21992-22068},22068,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21989,21992,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21225,361_Default/clip_grad_norm-ClipGradNorm/SumExt-op11,Default/clip_grad_norm-ClipGradNorm/SumExt-op11,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21994,21999,0x12c224a0fa00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21228,363_Default/clip_grad_norm-ClipGradNorm/Square-op12,Default/clip_grad_norm-ClipGradNorm/Square-op12,kernel_graph0,{21997},21997,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21996,22082,0x12c18000ac00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21228,364_Default/clip_grad_norm-ClipGradNorm/SumExt-op12,Default/clip_grad_norm-ClipGradNorm/SumExt-op12,kernel_graph0,{22000-22068},22068,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 21997,22000,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21228,364_Default/clip_grad_norm-ClipGradNorm/SumExt-op12,Default/clip_grad_norm-ClipGradNorm/SumExt-op12,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22002,22007,0x12c224a0fa00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21229,366_Default/clip_grad_norm-ClipGradNorm/Square-op13,Default/clip_grad_norm-ClipGradNorm/Square-op13,kernel_graph0,{22005},22005,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22004,22083,0x12c18000ae00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21231,367_Default/clip_grad_norm-ClipGradNorm/SumExt-op13,Default/clip_grad_norm-ClipGradNorm/SumExt-op13,kernel_graph0,{22008-22068},22068,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22005,22008,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21231,367_Default/clip_grad_norm-ClipGradNorm/SumExt-op13,Default/clip_grad_norm-ClipGradNorm/SumExt-op13,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22010,22015,0x12c224a0fa00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21233,369_Default/clip_grad_norm-ClipGradNorm/Square-op14,Default/clip_grad_norm-ClipGradNorm/Square-op14,kernel_graph0,{22013},22013,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22012,22084,0x12c18000b000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21234,370_Default/clip_grad_norm-ClipGradNorm/SumExt-op14,Default/clip_grad_norm-ClipGradNorm/SumExt-op14,kernel_graph0,{22016-22068},22068,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22013,22016,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21234,370_Default/clip_grad_norm-ClipGradNorm/SumExt-op14,Default/clip_grad_norm-ClipGradNorm/SumExt-op14,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22018,22023,0x12c1a6d36000,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21236,372_Default/clip_grad_norm-ClipGradNorm/Square-op15,Default/clip_grad_norm-ClipGradNorm/Square-op15,kernel_graph0,{22021},22021,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22020,22085,0x12c18000b200,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21237,373_Default/clip_grad_norm-ClipGradNorm/SumExt-op15,Default/clip_grad_norm-ClipGradNorm/SumExt-op15,kernel_graph0,{22024-22068},22068,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22021,22024,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,12800,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21237,373_Default/clip_grad_norm-ClipGradNorm/SumExt-op15,Default/clip_grad_norm-ClipGradNorm/SumExt-op15,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22026,22031,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21239,375_Default/clip_grad_norm-ClipGradNorm/Square-op16,Default/clip_grad_norm-ClipGradNorm/Square-op16,kernel_graph0,{22029},22029,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22028,22086,0x12c18000b400,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21240,376_Default/clip_grad_norm-ClipGradNorm/SumExt-op16,Default/clip_grad_norm-ClipGradNorm/SumExt-op16,kernel_graph0,{22032-22068},22068,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22029,22032,0x12c18000b600,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21240,376_Default/clip_grad_norm-ClipGradNorm/SumExt-op16,Default/clip_grad_norm-ClipGradNorm/SumExt-op16,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22034,22039,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21241,378_Default/clip_grad_norm-ClipGradNorm/Square-op17,Default/clip_grad_norm-ClipGradNorm/Square-op17,kernel_graph0,{22037},22037,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22036,22087,0x12c18000b600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21243,379_Default/clip_grad_norm-ClipGradNorm/SumExt-op17,Default/clip_grad_norm-ClipGradNorm/SumExt-op17,kernel_graph0,{22040-22068},22068,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22037,22040,0x12c18000b800,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21243,379_Default/clip_grad_norm-ClipGradNorm/SumExt-op17,Default/clip_grad_norm-ClipGradNorm/SumExt-op17,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22042,22047,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21244,381_Default/clip_grad_norm-ClipGradNorm/Square-op18,Default/clip_grad_norm-ClipGradNorm/Square-op18,kernel_graph0,{22045},22045,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22044,22088,0x12c18000b800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21247,382_Default/clip_grad_norm-ClipGradNorm/SumExt-op18,Default/clip_grad_norm-ClipGradNorm/SumExt-op18,kernel_graph0,{22048-22068},22068,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22045,22048,0x12c18000ba00,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21247,382_Default/clip_grad_norm-ClipGradNorm/SumExt-op18,Default/clip_grad_norm-ClipGradNorm/SumExt-op18,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22050,22055,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21248,384_Default/clip_grad_norm-ClipGradNorm/Square-op19,Default/clip_grad_norm-ClipGradNorm/Square-op19,kernel_graph0,{22053},22053,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22052,22089,0x12c18000ba00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21248,385_Default/clip_grad_norm-ClipGradNorm/SumExt-op19,Default/clip_grad_norm-ClipGradNorm/SumExt-op19,kernel_graph0,{22056-22068},22068,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22053,22056,0x12c1f1c12800,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21248,385_Default/clip_grad_norm-ClipGradNorm/SumExt-op19,Default/clip_grad_norm-ClipGradNorm/SumExt-op19,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22058,9223372036854775807,0x12c18000bc00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21248,387_Default/AllFinite-op0,Default/AllFinite-op0,kernel_graph0,{22094-22097-22099},22099,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22060,22065,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21249,388_Default/clip_grad_norm-ClipGradNorm/Square-op20,Default/clip_grad_norm-ClipGradNorm/Square-op20,kernel_graph0,{22063},22063,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22062,22090,0x12c18000be00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21250,389_Default/clip_grad_norm-ClipGradNorm/SumExt-op20,Default/clip_grad_norm-ClipGradNorm/SumExt-op20,kernel_graph0,{22066-22068},22068,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22063,22066,0x12c1f1c12800,0,DefaultEnhancedAscendMemoryPool,1536,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21250,389_Default/clip_grad_norm-ClipGradNorm/SumExt-op20,Default/clip_grad_norm-ClipGradNorm/SumExt-op20,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22068,22093,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21252,391_Default/clip_grad_norm-ClipGradNorm/AddN-op0,Default/clip_grad_norm-ClipGradNorm/AddN-op0,kernel_graph0,{22091},22091,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22091,9223372036854775807,0x12c180008a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21252,392_Default/clip_grad_norm-ClipGradNorm/Sqrt-op0,Default/clip_grad_norm-ClipGradNorm/Sqrt-op0,kernel_graph0,{22101-22103},22103,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22094,22124,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21252,393_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Select-op1,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Select-op1,kernel_graph0,{22115-22120},22120,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22097,22241,0x12c180008c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21252,394_Default/LogicalNot-op0,Default/LogicalNot-op0,kernel_graph0,{22117},22117,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22099,22127,0x12c18000be00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21254,395_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/LogicalOr-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/LogicalOr-op0,kernel_graph0,{22125},22125,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22101,22105,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21255,396_Default/clip_grad_norm-ClipGradNorm/GreaterEqual-op0,Default/clip_grad_norm-ClipGradNorm/GreaterEqual-op0,kernel_graph0,{22103},22103,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22103,22108,0x12c180009a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21255,397_Default/clip_grad_norm-ClipGradNorm/Select-op0,Default/clip_grad_norm-ClipGradNorm/Select-op0,kernel_graph0,{22106},22106,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22106,22111,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21256,398_Default/learning_rate-CosineWithWarmUpLR/Add-op0,Default/learning_rate-CosineWithWarmUpLR/Add-op0,kernel_graph0,{22109},22109,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22109,22114,0x12c180009a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21257,399_Default/clip_grad_norm-ClipGradNorm/Div-op0,Default/clip_grad_norm-ClipGradNorm/Div-op0,kernel_graph0,{22112},22112,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22112,22193,0x12c180009800,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21257,400_Default/clip_grad_norm-ClipGradNorm/Minimum-op0,Default/clip_grad_norm-ClipGradNorm/Minimum-op0,kernel_graph0,{22130-22133-22136-22139-22142-22145-22148-22151-22154-22157-22160-22163-22166-22169-22172-22175-22178-22181-22184-22187-22190},22190,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22115,22123,0x12c180009a00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21259,401_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Mul-op1,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Mul-op1,kernel_graph0,{22120},22120,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22117,22122,0x12c180009c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21260,402_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/LogicalAnd-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/LogicalAnd-op0,kernel_graph0,{22120},22120,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22120,22199,0x12c180009600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21260,403_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Select-op2,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Select-op2,kernel_graph0,{22197},22197,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22125,22195,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21263,404_Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Select-op0,Default/loss_scaling_manager-DynamicLossScaleUpdateCell/Select-op0,kernel_graph0,{22193},22193,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22128,22197,0x12c18000be00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21264,405_Default/learning_rate-CosineWithWarmUpLR/Add-op1,Default/learning_rate-CosineWithWarmUpLR/Add-op1,kernel_graph0,{22195},22195,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22130,22400,0x12c1a6d36000,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21264,406_Default/clip_grad_norm-ClipGradNorm/Mul-op0,Default/clip_grad_norm-ClipGradNorm/Mul-op0,kernel_graph0,{22396-22398},22398,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22133,22390,0x12c209e0ee00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21266,407_Default/clip_grad_norm-ClipGradNorm/Mul-op1,Default/clip_grad_norm-ClipGradNorm/Mul-op1,kernel_graph0,{22386-22388},22388,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22136,22382,0x12c1f1c12800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21267,408_Default/clip_grad_norm-ClipGradNorm/Mul-op2,Default/clip_grad_norm-ClipGradNorm/Mul-op2,kernel_graph0,{22378-22380},22380,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22139,22374,0x12c1e9c0e200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21267,409_Default/clip_grad_norm-ClipGradNorm/Mul-op3,Default/clip_grad_norm-ClipGradNorm/Mul-op3,kernel_graph0,{22370-22372},22372,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22142,22366,0x12c1f5c12a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21270,410_Default/clip_grad_norm-ClipGradNorm/Mul-op4,Default/clip_grad_norm-ClipGradNorm/Mul-op4,kernel_graph0,{22362-22364},22364,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22145,22358,0x12c1c6136200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21270,411_Default/clip_grad_norm-ClipGradNorm/Mul-op5,Default/clip_grad_norm-ClipGradNorm/Mul-op5,kernel_graph0,{22354-22356},22356,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22148,22350,0x12c1d0d36400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21271,412_Default/clip_grad_norm-ClipGradNorm/Mul-op6,Default/clip_grad_norm-ClipGradNorm/Mul-op6,kernel_graph0,{22346-22348},22348,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22151,22342,0x12c20de0f000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21272,413_Default/clip_grad_norm-ClipGradNorm/Mul-op7,Default/clip_grad_norm-ClipGradNorm/Mul-op7,kernel_graph0,{22338-22340},22340,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22154,22334,0x12c23d610600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21273,414_Default/clip_grad_norm-ClipGradNorm/Mul-op8,Default/clip_grad_norm-ClipGradNorm/Mul-op8,kernel_graph0,{22330-22332},22332,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22157,22326,0x12c1db936600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21274,415_Default/clip_grad_norm-ClipGradNorm/Mul-op9,Default/clip_grad_norm-ClipGradNorm/Mul-op9,kernel_graph0,{22322-22324},22324,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22160,22318,0x12c1edc0e400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21274,416_Default/clip_grad_norm-ClipGradNorm/Mul-op10,Default/clip_grad_norm-ClipGradNorm/Mul-op10,kernel_graph0,{22314-22316},22316,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22163,22310,0x12c1df936800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21274,417_Default/clip_grad_norm-ClipGradNorm/Mul-op11,Default/clip_grad_norm-ClipGradNorm/Mul-op11,kernel_graph0,{22306-22308},22308,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22166,22302,0x12c224a0fa00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21274,418_Default/clip_grad_norm-ClipGradNorm/Mul-op12,Default/clip_grad_norm-ClipGradNorm/Mul-op12,kernel_graph0,{22298-22300},22300,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22169,22294,0x12c22f60fc00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21275,419_Default/clip_grad_norm-ClipGradNorm/Mul-op13,Default/clip_grad_norm-ClipGradNorm/Mul-op13,kernel_graph0,{22290-22292},22292,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22172,22286,0x12c218a0f200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21276,420_Default/clip_grad_norm-ClipGradNorm/Mul-op14,Default/clip_grad_norm-ClipGradNorm/Mul-op14,kernel_graph0,{22282-22284},22284,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22175,22278,0x12c180018a00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21276,421_Default/clip_grad_norm-ClipGradNorm/Mul-op15,Default/clip_grad_norm-ClipGradNorm/Mul-op15,kernel_graph0,{22274-22276},22276,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22178,22270,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21278,422_Default/clip_grad_norm-ClipGradNorm/Mul-op16,Default/clip_grad_norm-ClipGradNorm/Mul-op16,kernel_graph0,{22266-22268},22268,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22181,22262,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21280,423_Default/clip_grad_norm-ClipGradNorm/Mul-op17,Default/clip_grad_norm-ClipGradNorm/Mul-op17,kernel_graph0,{22258-22260},22260,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22184,22254,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21281,424_Default/clip_grad_norm-ClipGradNorm/Mul-op18,Default/clip_grad_norm-ClipGradNorm/Mul-op18,kernel_graph0,{22250-22252},22252,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22187,22246,0x12c180014800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21281,425_Default/clip_grad_norm-ClipGradNorm/Mul-op19,Default/clip_grad_norm-ClipGradNorm/Mul-op19,kernel_graph0,{22242-22244},22244,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22190,22408,0x12c180010600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21284,426_Default/clip_grad_norm-ClipGradNorm/Mul-op20,Default/clip_grad_norm-ClipGradNorm/Mul-op20,kernel_graph0,{22404-22406},22406,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22200,22209,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21285,430_Default/learning_rate-CosineWithWarmUpLR/Cast-op0,Default/learning_rate-CosineWithWarmUpLR/Cast-op0,kernel_graph0,{22202-22205-22207},22207,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22202,22204,0x12c18000be00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,21286,431_Default/learning_rate-CosineWithWarmUpLR/GreaterEqual-op0,Default/learning_rate-CosineWithWarmUpLR/GreaterEqual-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22205,22210,0x12c18000be00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22204,435_Default/learning_rate-CosineWithWarmUpLR/Greater-op0,Default/learning_rate-CosineWithWarmUpLR/Greater-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22207,22213,0x12c180009600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22205,436_Default/TensorMove-op0,Default/TensorMove-op0,kernel_graph0,{22211},22211,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22211,22216,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22210,440_Default/learning_rate-CosineWithWarmUpLR/Sub-op1,Default/learning_rate-CosineWithWarmUpLR/Sub-op1,kernel_graph0,{22214},22214,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22214,22219,0x12c18000be00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22211,441_Default/learning_rate-CosineWithWarmUpLR/Div-op1,Default/learning_rate-CosineWithWarmUpLR/Div-op1,kernel_graph0,{22217},22217,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22217,22222,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22212,442_Default/learning_rate-CosineWithWarmUpLR/Mul-op1,Default/learning_rate-CosineWithWarmUpLR/Mul-op1,kernel_graph0,{22220},22220,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22220,22225,0x12c18000be00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22212,443_Default/learning_rate-CosineWithWarmUpLR/Cos-op0,Default/learning_rate-CosineWithWarmUpLR/Cos-op0,kernel_graph0,{22223},22223,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22223,22228,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22214,444_Default/learning_rate-CosineWithWarmUpLR/Add-op3,Default/learning_rate-CosineWithWarmUpLR/Add-op3,kernel_graph0,{22226},22226,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22226,22231,0x12c18000be00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22215,445_Default/learning_rate-CosineWithWarmUpLR/Mul-op2,Default/learning_rate-CosineWithWarmUpLR/Mul-op2,kernel_graph0,{22229},22229,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22229,22234,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22215,446_Default/learning_rate-CosineWithWarmUpLR/Maximum-op0,Default/learning_rate-CosineWithWarmUpLR/Maximum-op0,kernel_graph0,{22232},22232,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22232,22237,0x12c18000be00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22217,447_Default/learning_rate-CosineWithWarmUpLR/Mul-op3,Default/learning_rate-CosineWithWarmUpLR/Mul-op3,kernel_graph0,{22235},22235,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22235,22240,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22218,448_Default/learning_rate-CosineWithWarmUpLR/Add-op4,Default/learning_rate-CosineWithWarmUpLR/Add-op4,kernel_graph0,{22238},22238,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22238,9223372036854775807,0x12c18000be00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22221,451_Default/Reshape-op0,Default/Reshape-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22242,23071,0x12c1f1c0e600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22241,453_Default/optimizer-FP32StateAdamWeightDecay/Mul-op96,Default/optimizer-FP32StateAdamWeightDecay/Mul-op96,kernel_graph0,{23068},23068,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22244,22249,0x12c22360f400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22242,454_Default/optimizer-FP32StateAdamWeightDecay/Square-op19,Default/optimizer-FP32StateAdamWeightDecay/Square-op19,kernel_graph0,{22247},22247,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22247,23067,0x12c180014800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22243,455_Default/optimizer-FP32StateAdamWeightDecay/Mul-op98,Default/optimizer-FP32StateAdamWeightDecay/Mul-op98,kernel_graph0,{23064},23064,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22250,23035,0x12c22360f400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22243,456_Default/optimizer-FP32StateAdamWeightDecay/Mul-op91,Default/optimizer-FP32StateAdamWeightDecay/Mul-op91,kernel_graph0,{23032},23032,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22252,22257,0x12c223613600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22244,457_Default/optimizer-FP32StateAdamWeightDecay/Square-op18,Default/optimizer-FP32StateAdamWeightDecay/Square-op18,kernel_graph0,{22255},22255,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22255,23039,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22245,458_Default/optimizer-FP32StateAdamWeightDecay/Mul-op93,Default/optimizer-FP32StateAdamWeightDecay/Mul-op93,kernel_graph0,{23036},23036,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22258,23003,0x12c223613600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22245,459_Default/optimizer-FP32StateAdamWeightDecay/Mul-op86,Default/optimizer-FP32StateAdamWeightDecay/Mul-op86,kernel_graph0,{23000},23000,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22260,22265,0x12c223617800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22247,460_Default/optimizer-FP32StateAdamWeightDecay/Square-op17,Default/optimizer-FP32StateAdamWeightDecay/Square-op17,kernel_graph0,{22263},22263,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22263,23007,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22248,461_Default/optimizer-FP32StateAdamWeightDecay/Mul-op88,Default/optimizer-FP32StateAdamWeightDecay/Mul-op88,kernel_graph0,{23004},23004,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22266,22971,0x12c223617800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22248,462_Default/optimizer-FP32StateAdamWeightDecay/Mul-op81,Default/optimizer-FP32StateAdamWeightDecay/Mul-op81,kernel_graph0,{22968},22968,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22268,22273,0x12c22361ba00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22250,463_Default/optimizer-FP32StateAdamWeightDecay/Square-op16,Default/optimizer-FP32StateAdamWeightDecay/Square-op16,kernel_graph0,{22271},22271,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22271,22975,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22251,464_Default/optimizer-FP32StateAdamWeightDecay/Mul-op83,Default/optimizer-FP32StateAdamWeightDecay/Mul-op83,kernel_graph0,{22972},22972,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22274,22939,0x12c241610800,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22251,465_Default/optimizer-FP32StateAdamWeightDecay/Mul-op76,Default/optimizer-FP32StateAdamWeightDecay/Mul-op76,kernel_graph0,{22936},22936,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22276,22281,0x12c260a10a00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22252,466_Default/optimizer-FP32StateAdamWeightDecay/Square-op15,Default/optimizer-FP32StateAdamWeightDecay/Square-op15,kernel_graph0,{22279},22279,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22279,22943,0x12c180018a00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22253,467_Default/optimizer-FP32StateAdamWeightDecay/Mul-op78,Default/optimizer-FP32StateAdamWeightDecay/Mul-op78,kernel_graph0,{22940},22940,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22282,22907,0x12c1f9c12c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22253,468_Default/optimizer-FP32StateAdamWeightDecay/Mul-op71,Default/optimizer-FP32StateAdamWeightDecay/Mul-op71,kernel_graph0,{22904},22904,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22284,22289,0x12c260a10a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22255,469_Default/optimizer-FP32StateAdamWeightDecay/Square-op14,Default/optimizer-FP32StateAdamWeightDecay/Square-op14,kernel_graph0,{22287},22287,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22287,22911,0x12c218a0f200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22256,470_Default/optimizer-FP32StateAdamWeightDecay/Mul-op73,Default/optimizer-FP32StateAdamWeightDecay/Mul-op73,kernel_graph0,{22908},22908,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22290,22875,0x12c260a10a00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22256,471_Default/optimizer-FP32StateAdamWeightDecay/Mul-op66,Default/optimizer-FP32StateAdamWeightDecay/Mul-op66,kernel_graph0,{22872},22872,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22292,22297,0x12c26b610c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22258,472_Default/optimizer-FP32StateAdamWeightDecay/Square-op13,Default/optimizer-FP32StateAdamWeightDecay/Square-op13,kernel_graph0,{22295},22295,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22295,22879,0x12c22f60fc00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22259,473_Default/optimizer-FP32StateAdamWeightDecay/Mul-op68,Default/optimizer-FP32StateAdamWeightDecay/Mul-op68,kernel_graph0,{22876},22876,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22298,22843,0x12c26b610c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22259,474_Default/optimizer-FP32StateAdamWeightDecay/Mul-op61,Default/optimizer-FP32StateAdamWeightDecay/Mul-op61,kernel_graph0,{22840},22840,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22300,22305,0x12c276210e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22260,475_Default/optimizer-FP32StateAdamWeightDecay/Square-op12,Default/optimizer-FP32StateAdamWeightDecay/Square-op12,kernel_graph0,{22303},22303,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22303,22847,0x12c22361ba00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22261,476_Default/optimizer-FP32StateAdamWeightDecay/Mul-op63,Default/optimizer-FP32StateAdamWeightDecay/Mul-op63,kernel_graph0,{22844},22844,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22306,22811,0x12c204812e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22261,477_Default/optimizer-FP32StateAdamWeightDecay/Mul-op56,Default/optimizer-FP32StateAdamWeightDecay/Mul-op56,kernel_graph0,{22808},22808,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22308,22313,0x12c1e3936a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22263,478_Default/optimizer-FP32StateAdamWeightDecay/Square-op11,Default/optimizer-FP32StateAdamWeightDecay/Square-op11,kernel_graph0,{22311},22311,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22311,22815,0x12c1df936800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22264,479_Default/optimizer-FP32StateAdamWeightDecay/Mul-op58,Default/optimizer-FP32StateAdamWeightDecay/Mul-op58,kernel_graph0,{22812},22812,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22314,22779,0x12c1e3936a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22264,480_Default/optimizer-FP32StateAdamWeightDecay/Mul-op51,Default/optimizer-FP32StateAdamWeightDecay/Mul-op51,kernel_graph0,{22776},22776,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22316,22321,0x12c19f418c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22266,481_Default/optimizer-FP32StateAdamWeightDecay/Square-op10,Default/optimizer-FP32StateAdamWeightDecay/Square-op10,kernel_graph0,{22319},22319,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22319,22783,0x12c1edc0e400,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22267,482_Default/optimizer-FP32StateAdamWeightDecay/Mul-op53,Default/optimizer-FP32StateAdamWeightDecay/Mul-op53,kernel_graph0,{22780},22780,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22322,22747,0x12c19f418c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22267,483_Default/optimizer-FP32StateAdamWeightDecay/Mul-op46,Default/optimizer-FP32StateAdamWeightDecay/Mul-op46,kernel_graph0,{22744},22744,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22324,22329,0x12c276210e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22268,484_Default/optimizer-FP32StateAdamWeightDecay/Square-op9,Default/optimizer-FP32StateAdamWeightDecay/Square-op9,kernel_graph0,{22327},22327,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22327,22751,0x12c1db936600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22269,485_Default/optimizer-FP32StateAdamWeightDecay/Mul-op48,Default/optimizer-FP32StateAdamWeightDecay/Mul-op48,kernel_graph0,{22748},22748,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22330,22715,0x12c276210e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22269,486_Default/optimizer-FP32StateAdamWeightDecay/Mul-op41,Default/optimizer-FP32StateAdamWeightDecay/Mul-op41,kernel_graph0,{22712},22712,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22332,22337,0x12c27a211000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22271,487_Default/optimizer-FP32StateAdamWeightDecay/Square-op8,Default/optimizer-FP32StateAdamWeightDecay/Square-op8,kernel_graph0,{22335},22335,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22335,22719,0x12c23a20fe00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22272,488_Default/optimizer-FP32StateAdamWeightDecay/Mul-op43,Default/optimizer-FP32StateAdamWeightDecay/Mul-op43,kernel_graph0,{22716},22716,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22338,22683,0x12c27a211000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22272,489_Default/optimizer-FP32StateAdamWeightDecay/Mul-op36,Default/optimizer-FP32StateAdamWeightDecay/Mul-op36,kernel_graph0,{22680},22680,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22340,22345,0x12c284e11200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22274,490_Default/optimizer-FP32StateAdamWeightDecay/Square-op7,Default/optimizer-FP32StateAdamWeightDecay/Square-op7,kernel_graph0,{22343},22343,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22343,22687,0x12c20de0f000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22275,491_Default/optimizer-FP32StateAdamWeightDecay/Mul-op38,Default/optimizer-FP32StateAdamWeightDecay/Mul-op38,kernel_graph0,{22684},22684,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22346,22651,0x12c284e11200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22275,492_Default/optimizer-FP32StateAdamWeightDecay/Mul-op31,Default/optimizer-FP32StateAdamWeightDecay/Mul-op31,kernel_graph0,{22648},22648,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22348,22353,0x12c28fa11400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22276,493_Default/optimizer-FP32StateAdamWeightDecay/Square-op6,Default/optimizer-FP32StateAdamWeightDecay/Square-op6,kernel_graph0,{22351},22351,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22351,22655,0x12c1d0d36400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22277,494_Default/optimizer-FP32StateAdamWeightDecay/Mul-op33,Default/optimizer-FP32StateAdamWeightDecay/Mul-op33,kernel_graph0,{22652},22652,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22354,22619,0x12c28fa11400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22277,495_Default/optimizer-FP32StateAdamWeightDecay/Mul-op26,Default/optimizer-FP32StateAdamWeightDecay/Mul-op26,kernel_graph0,{22616},22616,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22356,22361,0x12c29a611600,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22279,496_Default/optimizer-FP32StateAdamWeightDecay/Square-op5,Default/optimizer-FP32StateAdamWeightDecay/Square-op5,kernel_graph0,{22359},22359,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22359,22623,0x12c1c6136200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22280,497_Default/optimizer-FP32StateAdamWeightDecay/Mul-op28,Default/optimizer-FP32StateAdamWeightDecay/Mul-op28,kernel_graph0,{22620},22620,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22362,22587,0x12c29a611600,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22280,498_Default/optimizer-FP32StateAdamWeightDecay/Mul-op21,Default/optimizer-FP32StateAdamWeightDecay/Mul-op21,kernel_graph0,{22584},22584,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22364,22369,0x12c29e611800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22282,499_Default/optimizer-FP32StateAdamWeightDecay/Square-op4,Default/optimizer-FP32StateAdamWeightDecay/Square-op4,kernel_graph0,{22367},22367,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22367,22591,0x12c1f5c12a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22283,500_Default/optimizer-FP32StateAdamWeightDecay/Mul-op23,Default/optimizer-FP32StateAdamWeightDecay/Mul-op23,kernel_graph0,{22588},22588,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22370,22555,0x12c29e611800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22283,501_Default/optimizer-FP32StateAdamWeightDecay/Mul-op16,Default/optimizer-FP32StateAdamWeightDecay/Mul-op16,kernel_graph0,{22552},22552,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22372,22377,0x12c2a2611a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22284,502_Default/optimizer-FP32StateAdamWeightDecay/Square-op3,Default/optimizer-FP32StateAdamWeightDecay/Square-op3,kernel_graph0,{22375},22375,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22375,22559,0x12c1e7936c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22285,503_Default/optimizer-FP32StateAdamWeightDecay/Mul-op18,Default/optimizer-FP32StateAdamWeightDecay/Mul-op18,kernel_graph0,{22556},22556,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22378,22523,0x12c2a2611a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22285,504_Default/optimizer-FP32StateAdamWeightDecay/Mul-op11,Default/optimizer-FP32StateAdamWeightDecay/Mul-op11,kernel_graph0,{22520},22520,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22380,22385,0x12c2a6611c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22287,505_Default/optimizer-FP32StateAdamWeightDecay/Square-op2,Default/optimizer-FP32StateAdamWeightDecay/Square-op2,kernel_graph0,{22383},22383,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22383,22527,0x12c1f1c12800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22288,506_Default/optimizer-FP32StateAdamWeightDecay/Mul-op13,Default/optimizer-FP32StateAdamWeightDecay/Mul-op13,kernel_graph0,{22524},22524,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22386,22491,0x12c2a6611c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22288,507_Default/optimizer-FP32StateAdamWeightDecay/Mul-op6,Default/optimizer-FP32StateAdamWeightDecay/Mul-op6,kernel_graph0,{22488},22488,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22388,22393,0x12c2aa611e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22290,508_Default/optimizer-FP32StateAdamWeightDecay/Square-op1,Default/optimizer-FP32StateAdamWeightDecay/Square-op1,kernel_graph0,{22391},22391,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22391,22495,0x12c208813000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22291,509_Default/optimizer-FP32StateAdamWeightDecay/Mul-op8,Default/optimizer-FP32StateAdamWeightDecay/Mul-op8,kernel_graph0,{22492},22492,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22394,22419,0x12c180008c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22291,510_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Cast-op0,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Cast-op0,kernel_graph0,{22412-22415-22417},22417,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22396,22459,0x12c2aa611e00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22292,511_Default/optimizer-FP32StateAdamWeightDecay/Mul-op1,Default/optimizer-FP32StateAdamWeightDecay/Mul-op1,kernel_graph0,{22456},22456,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22398,22403,0x12c2c9a12000,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22293,512_Default/optimizer-FP32StateAdamWeightDecay/Square-op0,Default/optimizer-FP32StateAdamWeightDecay/Square-op0,kernel_graph0,{22401},22401,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22401,22463,0x12c1a3418e00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22293,513_Default/optimizer-FP32StateAdamWeightDecay/Mul-op3,Default/optimizer-FP32StateAdamWeightDecay/Mul-op3,kernel_graph0,{22460},22460,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22404,23103,0x12c22e21bc00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22295,514_Default/optimizer-FP32StateAdamWeightDecay/Mul-op101,Default/optimizer-FP32StateAdamWeightDecay/Mul-op101,kernel_graph0,{23100},23100,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22406,22411,0x12c22e21fe00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22296,515_Default/optimizer-FP32StateAdamWeightDecay/Square-op20,Default/optimizer-FP32StateAdamWeightDecay/Square-op20,kernel_graph0,{22409},22409,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22409,23099,0x12c180010600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22296,516_Default/optimizer-FP32StateAdamWeightDecay/Mul-op103,Default/optimizer-FP32StateAdamWeightDecay/Mul-op103,kernel_graph0,{23096},23096,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22412,22414,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22298,518_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/GreaterEqual-op0,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/GreaterEqual-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22415,22420,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22414,522_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Greater-op0,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Greater-op0,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22417,22423,0x12c180009600,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22415,523_Default/TensorMove-op1,Default/TensorMove-op1,kernel_graph0,{22421},22421,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22421,22426,0x12c180008c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22420,527_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Sub-op1,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Sub-op1,kernel_graph0,{22424},22424,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22424,22429,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22421,528_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Div-op1,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Div-op1,kernel_graph0,{22427},22427,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22427,22432,0x12c180008c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22422,529_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Mul-op1,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Mul-op1,kernel_graph0,{22430},22430,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22430,22435,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22422,530_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Cos-op0,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Cos-op0,kernel_graph0,{22433},22433,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22433,22438,0x12c180008c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22424,531_Default/learning_rate-CosineWithWarmUpLR/Add-op6,Default/learning_rate-CosineWithWarmUpLR/Add-op6,kernel_graph0,{22436},22436,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22436,22441,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22425,532_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Mul-op2,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Mul-op2,kernel_graph0,{22439},22439,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22439,22444,0x12c180008c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22425,533_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Maximum-op0,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Maximum-op0,kernel_graph0,{22442},22442,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22442,22447,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22427,534_Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Mul-op3,Default/optimizer-FP32StateAdamWeightDecay/learning_rate-CosineWithWarmUpLR/Mul-op3,kernel_graph0,{22445},22445,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22445,22450,0x12c180008c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22428,535_Default/learning_rate-CosineWithWarmUpLR/Add-op7,Default/learning_rate-CosineWithWarmUpLR/Add-op7,kernel_graph0,{22448},22448,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22448,23114,0x12c180009000,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22431,538_Default/optimizer-FP32StateAdamWeightDecay/Reshape-op0,Default/optimizer-FP32StateAdamWeightDecay/Reshape-op0,kernel_graph0,{22472-22504-22536-22568-22600-22632-22664-22696-22728-22760-22792-22824-22856-22888-22920-22952-22984-23016-23048-23080-23112},23112,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22452,22458,0x12c2c9a12000,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22433,540_Default/optimizer-FP32StateAdamWeightDecay/Mul-op0,Default/optimizer-FP32StateAdamWeightDecay/Mul-op0,kernel_graph0,{22456},22456,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22454,22462,0x12c2e8e12200,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22434,541_Default/optimizer-FP32StateAdamWeightDecay/Mul-op2,Default/optimizer-FP32StateAdamWeightDecay/Mul-op2,kernel_graph0,{22460},22460,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22456,22481,0x12c308212400,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22434,542_Default/learning_rate-CosineWithWarmUpLR/Add-op8,Default/learning_rate-CosineWithWarmUpLR/Add-op8,kernel_graph0,{22469-22479},22479,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22460,22479,0x12c2aa611e00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22436,543_Default/learning_rate-CosineWithWarmUpLR/Add-op9,Default/learning_rate-CosineWithWarmUpLR/Add-op9,kernel_graph0,{22464-22477},22477,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22464,22468,0x12c1a3418e00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22437,544_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op0,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op0,kernel_graph0,{22466},22466,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22466,22471,0x12c2c9a12000,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22437,545_Default/learning_rate-CosineWithWarmUpLR/Add-op10,Default/learning_rate-CosineWithWarmUpLR/Add-op10,kernel_graph0,{22469},22469,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22469,22474,0x12c2e8e12200,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22439,546_Default/optimizer-FP32StateAdamWeightDecay/Div-op0,Default/optimizer-FP32StateAdamWeightDecay/Div-op0,kernel_graph0,{22472},22472,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22472,22477,0x12c2c9a12000,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22440,547_Default/optimizer-FP32StateAdamWeightDecay/Mul-op4,Default/optimizer-FP32StateAdamWeightDecay/Mul-op4,kernel_graph0,{22475},22475,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22475,22483,0x12c2e8e12200,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22440,548_Default/optimizer-FP32StateAdamWeightDecay/Sub-op0,Default/optimizer-FP32StateAdamWeightDecay/Sub-op0,kernel_graph0,{22481},22481,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22484,22490,0x12c1a3418e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22445,552_Default/optimizer-FP32StateAdamWeightDecay/Mul-op5,Default/optimizer-FP32StateAdamWeightDecay/Mul-op5,kernel_graph0,{22488},22488,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22486,22494,0x12c1a7419000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22446,553_Default/optimizer-FP32StateAdamWeightDecay/Mul-op7,Default/optimizer-FP32StateAdamWeightDecay/Mul-op7,kernel_graph0,{22492},22492,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22488,22513,0x12c1ab419200,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22446,554_Default/learning_rate-CosineWithWarmUpLR/Add-op11,Default/learning_rate-CosineWithWarmUpLR/Add-op11,kernel_graph0,{22501-22511},22511,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22492,22511,0x12c1a3418e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22448,555_Default/learning_rate-CosineWithWarmUpLR/Add-op12,Default/learning_rate-CosineWithWarmUpLR/Add-op12,kernel_graph0,{22496-22509},22509,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22496,22500,0x12c1a7419000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22449,556_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op1,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op1,kernel_graph0,{22498},22498,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22498,22503,0x12c208813000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22450,557_Default/learning_rate-CosineWithWarmUpLR/Add-op13,Default/learning_rate-CosineWithWarmUpLR/Add-op13,kernel_graph0,{22501},22501,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22501,22506,0x12c1a7419000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22450,558_Default/optimizer-FP32StateAdamWeightDecay/Div-op1,Default/optimizer-FP32StateAdamWeightDecay/Div-op1,kernel_graph0,{22504},22504,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22504,22509,0x12c208813000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22451,559_Default/optimizer-FP32StateAdamWeightDecay/Mul-op9,Default/optimizer-FP32StateAdamWeightDecay/Mul-op9,kernel_graph0,{22507},22507,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22507,22515,0x12c1a7419000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22451,560_Default/optimizer-FP32StateAdamWeightDecay/Sub-op1,Default/optimizer-FP32StateAdamWeightDecay/Sub-op1,kernel_graph0,{22513},22513,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22516,22522,0x12c208813000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22453,564_Default/optimizer-FP32StateAdamWeightDecay/Mul-op10,Default/optimizer-FP32StateAdamWeightDecay/Mul-op10,kernel_graph0,{22520},22520,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22518,22526,0x12c1a3418e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22453,565_Default/optimizer-FP32StateAdamWeightDecay/Mul-op12,Default/optimizer-FP32StateAdamWeightDecay/Mul-op12,kernel_graph0,{22524},22524,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22520,22545,0x12c1a7419000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22454,566_Default/learning_rate-CosineWithWarmUpLR/Add-op14,Default/learning_rate-CosineWithWarmUpLR/Add-op14,kernel_graph0,{22533-22543},22543,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22524,22543,0x12c208813000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22455,567_Default/learning_rate-CosineWithWarmUpLR/Add-op15,Default/learning_rate-CosineWithWarmUpLR/Add-op15,kernel_graph0,{22528-22541},22541,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22528,22532,0x12c1a3418e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22456,568_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op2,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op2,kernel_graph0,{22530},22530,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22530,22535,0x12c1f1c12800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22456,569_Default/learning_rate-CosineWithWarmUpLR/Add-op16,Default/learning_rate-CosineWithWarmUpLR/Add-op16,kernel_graph0,{22533},22533,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22533,22538,0x12c1a3418e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22457,570_Default/optimizer-FP32StateAdamWeightDecay/Div-op2,Default/optimizer-FP32StateAdamWeightDecay/Div-op2,kernel_graph0,{22536},22536,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22536,22541,0x12c1f1c12800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22459,571_Default/optimizer-FP32StateAdamWeightDecay/Mul-op14,Default/optimizer-FP32StateAdamWeightDecay/Mul-op14,kernel_graph0,{22539},22539,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22539,22547,0x12c1a3418e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22460,572_Default/optimizer-FP32StateAdamWeightDecay/Sub-op2,Default/optimizer-FP32StateAdamWeightDecay/Sub-op2,kernel_graph0,{22545},22545,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22548,22554,0x12c1f1c12800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22465,576_Default/optimizer-FP32StateAdamWeightDecay/Mul-op15,Default/optimizer-FP32StateAdamWeightDecay/Mul-op15,kernel_graph0,{22552},22552,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22550,22558,0x12c208813000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22465,577_Default/optimizer-FP32StateAdamWeightDecay/Mul-op17,Default/optimizer-FP32StateAdamWeightDecay/Mul-op17,kernel_graph0,{22556},22556,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22552,22577,0x12c1a3418e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22466,578_Default/learning_rate-CosineWithWarmUpLR/Add-op17,Default/learning_rate-CosineWithWarmUpLR/Add-op17,kernel_graph0,{22565-22575},22575,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22556,22575,0x12c1f1c12800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22467,579_Default/learning_rate-CosineWithWarmUpLR/Add-op18,Default/learning_rate-CosineWithWarmUpLR/Add-op18,kernel_graph0,{22560-22573},22573,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22560,22564,0x12c208813000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22467,580_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op3,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op3,kernel_graph0,{22562},22562,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22562,22567,0x12c1e7936c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22469,581_Default/learning_rate-CosineWithWarmUpLR/Add-op19,Default/learning_rate-CosineWithWarmUpLR/Add-op19,kernel_graph0,{22565},22565,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22565,22570,0x12c208813000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22470,582_Default/optimizer-FP32StateAdamWeightDecay/Div-op3,Default/optimizer-FP32StateAdamWeightDecay/Div-op3,kernel_graph0,{22568},22568,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22568,22573,0x12c1e7936c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22470,583_Default/optimizer-FP32StateAdamWeightDecay/Mul-op19,Default/optimizer-FP32StateAdamWeightDecay/Mul-op19,kernel_graph0,{22571},22571,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22571,22579,0x12c208813000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22472,584_Default/optimizer-FP32StateAdamWeightDecay/Sub-op3,Default/optimizer-FP32StateAdamWeightDecay/Sub-op3,kernel_graph0,{22577},22577,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22580,22586,0x12c1f1c12800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22476,588_Default/optimizer-FP32StateAdamWeightDecay/Mul-op20,Default/optimizer-FP32StateAdamWeightDecay/Mul-op20,kernel_graph0,{22584},22584,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22582,22590,0x12c208813000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22476,589_Default/optimizer-FP32StateAdamWeightDecay/Mul-op22,Default/optimizer-FP32StateAdamWeightDecay/Mul-op22,kernel_graph0,{22588},22588,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22584,22609,0x12c1e7936c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22477,590_Default/learning_rate-CosineWithWarmUpLR/Add-op20,Default/learning_rate-CosineWithWarmUpLR/Add-op20,kernel_graph0,{22597-22607},22607,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22588,22607,0x12c1f1c12800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22478,591_Default/learning_rate-CosineWithWarmUpLR/Add-op21,Default/learning_rate-CosineWithWarmUpLR/Add-op21,kernel_graph0,{22592-22605},22605,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22592,22596,0x12c1f5c12a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22478,592_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op4,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op4,kernel_graph0,{22594},22594,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22594,22599,0x12c208813000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22478,593_Default/learning_rate-CosineWithWarmUpLR/Add-op22,Default/learning_rate-CosineWithWarmUpLR/Add-op22,kernel_graph0,{22597},22597,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22597,22602,0x12c1f5c12a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22478,594_Default/optimizer-FP32StateAdamWeightDecay/Div-op4,Default/optimizer-FP32StateAdamWeightDecay/Div-op4,kernel_graph0,{22600},22600,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22600,22605,0x12c208813000,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22478,595_Default/optimizer-FP32StateAdamWeightDecay/Mul-op24,Default/optimizer-FP32StateAdamWeightDecay/Mul-op24,kernel_graph0,{22603},22603,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22603,22611,0x12c1f5c12a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22479,596_Default/optimizer-FP32StateAdamWeightDecay/Sub-op4,Default/optimizer-FP32StateAdamWeightDecay/Sub-op4,kernel_graph0,{22609},22609,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22612,22618,0x12c1a3418e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22481,600_Default/optimizer-FP32StateAdamWeightDecay/Mul-op25,Default/optimizer-FP32StateAdamWeightDecay/Mul-op25,kernel_graph0,{22616},22616,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22614,22622,0x12c1ae019000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22482,601_Default/optimizer-FP32StateAdamWeightDecay/Mul-op27,Default/optimizer-FP32StateAdamWeightDecay/Mul-op27,kernel_graph0,{22620},22620,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22616,22641,0x12c1b8c19200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22482,602_Default/learning_rate-CosineWithWarmUpLR/Add-op23,Default/learning_rate-CosineWithWarmUpLR/Add-op23,kernel_graph0,{22629-22639},22639,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22620,22639,0x12c1a3418e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22482,603_Default/learning_rate-CosineWithWarmUpLR/Add-op24,Default/learning_rate-CosineWithWarmUpLR/Add-op24,kernel_graph0,{22624-22637},22637,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22624,22628,0x12c1ae019000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22483,604_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op5,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op5,kernel_graph0,{22626},22626,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22626,22631,0x12c1c3819400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22484,605_Default/learning_rate-CosineWithWarmUpLR/Add-op25,Default/learning_rate-CosineWithWarmUpLR/Add-op25,kernel_graph0,{22629},22629,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22629,22634,0x12c1ae019000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22485,606_Default/optimizer-FP32StateAdamWeightDecay/Div-op5,Default/optimizer-FP32StateAdamWeightDecay/Div-op5,kernel_graph0,{22632},22632,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22632,22637,0x12c1c3819400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22485,607_Default/optimizer-FP32StateAdamWeightDecay/Mul-op29,Default/optimizer-FP32StateAdamWeightDecay/Mul-op29,kernel_graph0,{22635},22635,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22635,22643,0x12c1ae019000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22486,608_Default/optimizer-FP32StateAdamWeightDecay/Sub-op5,Default/optimizer-FP32StateAdamWeightDecay/Sub-op5,kernel_graph0,{22641},22641,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22644,22650,0x12c1a3418e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22489,612_Default/optimizer-FP32StateAdamWeightDecay/Mul-op30,Default/optimizer-FP32StateAdamWeightDecay/Mul-op30,kernel_graph0,{22648},22648,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22646,22654,0x12c1ae019000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22491,613_Default/optimizer-FP32StateAdamWeightDecay/Mul-op32,Default/optimizer-FP32StateAdamWeightDecay/Mul-op32,kernel_graph0,{22652},22652,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22648,22673,0x12c1b8c19200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22492,614_Default/learning_rate-CosineWithWarmUpLR/Add-op26,Default/learning_rate-CosineWithWarmUpLR/Add-op26,kernel_graph0,{22661-22671},22671,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22652,22671,0x12c1a3418e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22493,615_Default/learning_rate-CosineWithWarmUpLR/Add-op27,Default/learning_rate-CosineWithWarmUpLR/Add-op27,kernel_graph0,{22656-22669},22669,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22656,22660,0x12c1ae019000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22495,616_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op6,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op6,kernel_graph0,{22658},22658,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22658,22663,0x12c1c3819400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22496,617_Default/learning_rate-CosineWithWarmUpLR/Add-op28,Default/learning_rate-CosineWithWarmUpLR/Add-op28,kernel_graph0,{22661},22661,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22661,22666,0x12c1ae019000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22497,618_Default/optimizer-FP32StateAdamWeightDecay/Div-op6,Default/optimizer-FP32StateAdamWeightDecay/Div-op6,kernel_graph0,{22664},22664,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22664,22669,0x12c1c3819400,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22498,619_Default/optimizer-FP32StateAdamWeightDecay/Mul-op34,Default/optimizer-FP32StateAdamWeightDecay/Mul-op34,kernel_graph0,{22667},22667,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22667,22675,0x12c1ae019000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22498,620_Default/optimizer-FP32StateAdamWeightDecay/Sub-op6,Default/optimizer-FP32StateAdamWeightDecay/Sub-op6,kernel_graph0,{22673},22673,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22676,22682,0x12c1a3418e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22502,624_Default/optimizer-FP32StateAdamWeightDecay/Mul-op35,Default/optimizer-FP32StateAdamWeightDecay/Mul-op35,kernel_graph0,{22680},22680,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22678,22686,0x12c1ae019000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22503,625_Default/optimizer-FP32StateAdamWeightDecay/Mul-op37,Default/optimizer-FP32StateAdamWeightDecay/Mul-op37,kernel_graph0,{22684},22684,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22680,22705,0x12c1b8c19200,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22504,626_Default/learning_rate-CosineWithWarmUpLR/Add-op29,Default/learning_rate-CosineWithWarmUpLR/Add-op29,kernel_graph0,{22693-22703},22703,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22684,22703,0x12c1a3418e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22505,627_Default/learning_rate-CosineWithWarmUpLR/Add-op30,Default/learning_rate-CosineWithWarmUpLR/Add-op30,kernel_graph0,{22688-22701},22701,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22688,22692,0x12c1ae019000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22506,628_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op7,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op7,kernel_graph0,{22690},22690,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22690,22695,0x12c208813000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22507,629_Default/learning_rate-CosineWithWarmUpLR/Add-op31,Default/learning_rate-CosineWithWarmUpLR/Add-op31,kernel_graph0,{22693},22693,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22693,22698,0x12c1ae019000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22508,630_Default/optimizer-FP32StateAdamWeightDecay/Div-op7,Default/optimizer-FP32StateAdamWeightDecay/Div-op7,kernel_graph0,{22696},22696,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22696,22701,0x12c208813000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22509,631_Default/optimizer-FP32StateAdamWeightDecay/Mul-op39,Default/optimizer-FP32StateAdamWeightDecay/Mul-op39,kernel_graph0,{22699},22699,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22699,22707,0x12c1ae019000,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22509,632_Default/optimizer-FP32StateAdamWeightDecay/Sub-op7,Default/optimizer-FP32StateAdamWeightDecay/Sub-op7,kernel_graph0,{22705},22705,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22708,22714,0x12c1e7936c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22511,636_Default/optimizer-FP32StateAdamWeightDecay/Mul-op40,Default/optimizer-FP32StateAdamWeightDecay/Mul-op40,kernel_graph0,{22712},22712,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22710,22718,0x12c1f1c12800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22512,637_Default/optimizer-FP32StateAdamWeightDecay/Mul-op42,Default/optimizer-FP32StateAdamWeightDecay/Mul-op42,kernel_graph0,{22716},22716,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22712,22737,0x12c1f5c12a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22512,638_Default/learning_rate-CosineWithWarmUpLR/Add-op32,Default/learning_rate-CosineWithWarmUpLR/Add-op32,kernel_graph0,{22725-22735},22735,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22716,22735,0x12c1e7936c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22512,639_Default/learning_rate-CosineWithWarmUpLR/Add-op33,Default/learning_rate-CosineWithWarmUpLR/Add-op33,kernel_graph0,{22720-22733},22733,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22720,22724,0x12c1f1c12800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22513,640_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op8,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op8,kernel_graph0,{22722},22722,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22722,22727,0x12c23a20fe00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22513,641_Default/learning_rate-CosineWithWarmUpLR/Add-op34,Default/learning_rate-CosineWithWarmUpLR/Add-op34,kernel_graph0,{22725},22725,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22725,22730,0x12c1f1c12800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22514,642_Default/optimizer-FP32StateAdamWeightDecay/Div-op8,Default/optimizer-FP32StateAdamWeightDecay/Div-op8,kernel_graph0,{22728},22728,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22728,22733,0x12c23a20fe00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22514,643_Default/optimizer-FP32StateAdamWeightDecay/Mul-op44,Default/optimizer-FP32StateAdamWeightDecay/Mul-op44,kernel_graph0,{22731},22731,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22731,22739,0x12c1f1c12800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22514,644_Default/optimizer-FP32StateAdamWeightDecay/Sub-op8,Default/optimizer-FP32StateAdamWeightDecay/Sub-op8,kernel_graph0,{22737},22737,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22740,22746,0x12c1e7936c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22518,648_Default/optimizer-FP32StateAdamWeightDecay/Mul-op45,Default/optimizer-FP32StateAdamWeightDecay/Mul-op45,kernel_graph0,{22744},22744,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22742,22750,0x12c23a20fe00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22519,649_Default/optimizer-FP32StateAdamWeightDecay/Mul-op47,Default/optimizer-FP32StateAdamWeightDecay/Mul-op47,kernel_graph0,{22748},22748,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22744,22769,0x12c1f1c12800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22519,650_Default/learning_rate-CosineWithWarmUpLR/Add-op35,Default/learning_rate-CosineWithWarmUpLR/Add-op35,kernel_graph0,{22757-22767},22767,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22748,22767,0x12c1f5c12a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22520,651_Default/learning_rate-CosineWithWarmUpLR/Add-op36,Default/learning_rate-CosineWithWarmUpLR/Add-op36,kernel_graph0,{22752-22765},22765,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22752,22756,0x12c1e7936c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22521,652_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op9,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op9,kernel_graph0,{22754},22754,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22754,22759,0x12c23a20fe00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22521,653_Default/learning_rate-CosineWithWarmUpLR/Add-op37,Default/learning_rate-CosineWithWarmUpLR/Add-op37,kernel_graph0,{22757},22757,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22757,22762,0x12c1e7936c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22524,654_Default/optimizer-FP32StateAdamWeightDecay/Div-op9,Default/optimizer-FP32StateAdamWeightDecay/Div-op9,kernel_graph0,{22760},22760,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22760,22765,0x12c23a20fe00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22525,655_Default/optimizer-FP32StateAdamWeightDecay/Mul-op49,Default/optimizer-FP32StateAdamWeightDecay/Mul-op49,kernel_graph0,{22763},22763,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22763,22771,0x12c1e7936c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22525,656_Default/optimizer-FP32StateAdamWeightDecay/Sub-op9,Default/optimizer-FP32StateAdamWeightDecay/Sub-op9,kernel_graph0,{22769},22769,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22772,22778,0x12c1e7936c00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22530,660_Default/optimizer-FP32StateAdamWeightDecay/Mul-op50,Default/optimizer-FP32StateAdamWeightDecay/Mul-op50,kernel_graph0,{22776},22776,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22774,22782,0x12c23a20fe00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22531,661_Default/optimizer-FP32StateAdamWeightDecay/Mul-op52,Default/optimizer-FP32StateAdamWeightDecay/Mul-op52,kernel_graph0,{22780},22780,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22776,22801,0x12c1f1c12800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22532,662_Default/learning_rate-CosineWithWarmUpLR/Add-op38,Default/learning_rate-CosineWithWarmUpLR/Add-op38,kernel_graph0,{22789-22799},22799,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22780,22799,0x12c1f5c12a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22533,663_Default/learning_rate-CosineWithWarmUpLR/Add-op39,Default/learning_rate-CosineWithWarmUpLR/Add-op39,kernel_graph0,{22784-22797},22797,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22784,22788,0x12c23a20fe00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22534,664_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op10,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op10,kernel_graph0,{22786},22786,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22786,22791,0x12c1e3936a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22535,665_Default/learning_rate-CosineWithWarmUpLR/Add-op40,Default/learning_rate-CosineWithWarmUpLR/Add-op40,kernel_graph0,{22789},22789,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22789,22794,0x12c23a20fe00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22536,666_Default/optimizer-FP32StateAdamWeightDecay/Div-op10,Default/optimizer-FP32StateAdamWeightDecay/Div-op10,kernel_graph0,{22792},22792,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22792,22797,0x12c1e3936a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22537,667_Default/optimizer-FP32StateAdamWeightDecay/Mul-op54,Default/optimizer-FP32StateAdamWeightDecay/Mul-op54,kernel_graph0,{22795},22795,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22795,22803,0x12c23a20fe00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22539,668_Default/optimizer-FP32StateAdamWeightDecay/Sub-op10,Default/optimizer-FP32StateAdamWeightDecay/Sub-op10,kernel_graph0,{22801},22801,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22804,22810,0x12c23a20fe00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22542,672_Default/optimizer-FP32StateAdamWeightDecay/Mul-op55,Default/optimizer-FP32StateAdamWeightDecay/Mul-op55,kernel_graph0,{22808},22808,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22806,22814,0x12c1f1c12800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22542,673_Default/optimizer-FP32StateAdamWeightDecay/Mul-op57,Default/optimizer-FP32StateAdamWeightDecay/Mul-op57,kernel_graph0,{22812},22812,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22808,22833,0x12c1f5c12a00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22542,674_Default/learning_rate-CosineWithWarmUpLR/Add-op41,Default/learning_rate-CosineWithWarmUpLR/Add-op41,kernel_graph0,{22821-22831},22831,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22812,22831,0x12c23a20fe00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22543,675_Default/learning_rate-CosineWithWarmUpLR/Add-op42,Default/learning_rate-CosineWithWarmUpLR/Add-op42,kernel_graph0,{22816-22829},22829,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22816,22820,0x12c1f1c12800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22543,676_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op11,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op11,kernel_graph0,{22818},22818,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22818,22823,0x12c204812e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22544,677_Default/learning_rate-CosineWithWarmUpLR/Add-op43,Default/learning_rate-CosineWithWarmUpLR/Add-op43,kernel_graph0,{22821},22821,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22821,22826,0x12c1f1c12800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22544,678_Default/optimizer-FP32StateAdamWeightDecay/Div-op11,Default/optimizer-FP32StateAdamWeightDecay/Div-op11,kernel_graph0,{22824},22824,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22824,22829,0x12c204812e00,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22544,679_Default/optimizer-FP32StateAdamWeightDecay/Mul-op59,Default/optimizer-FP32StateAdamWeightDecay/Mul-op59,kernel_graph0,{22827},22827,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22827,22835,0x12c1f1c12800,0,DefaultEnhancedAscendMemoryPool,67109376,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22545,680_Default/optimizer-FP32StateAdamWeightDecay/Sub-op11,Default/optimizer-FP32StateAdamWeightDecay/Sub-op11,kernel_graph0,{22833},22833,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22836,22842,0x12c204812e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22547,684_Default/optimizer-FP32StateAdamWeightDecay/Mul-op60,Default/optimizer-FP32StateAdamWeightDecay/Mul-op60,kernel_graph0,{22840},22840,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22838,22846,0x12c19f418c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22548,685_Default/optimizer-FP32StateAdamWeightDecay/Mul-op62,Default/optimizer-FP32StateAdamWeightDecay/Mul-op62,kernel_graph0,{22844},22844,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22840,22865,0x12c1aa018e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22549,686_Default/learning_rate-CosineWithWarmUpLR/Add-op44,Default/learning_rate-CosineWithWarmUpLR/Add-op44,kernel_graph0,{22853-22863},22863,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22844,22863,0x12c204812e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22549,687_Default/learning_rate-CosineWithWarmUpLR/Add-op45,Default/learning_rate-CosineWithWarmUpLR/Add-op45,kernel_graph0,{22848-22861},22861,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22848,22852,0x12c19f418c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22550,688_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op12,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op12,kernel_graph0,{22850},22850,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22850,22855,0x12c22361ba00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22551,689_Default/learning_rate-CosineWithWarmUpLR/Add-op46,Default/learning_rate-CosineWithWarmUpLR/Add-op46,kernel_graph0,{22853},22853,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22853,22858,0x12c19f418c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22551,690_Default/optimizer-FP32StateAdamWeightDecay/Div-op12,Default/optimizer-FP32StateAdamWeightDecay/Div-op12,kernel_graph0,{22856},22856,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22856,22861,0x12c22361ba00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22552,691_Default/optimizer-FP32StateAdamWeightDecay/Mul-op64,Default/optimizer-FP32StateAdamWeightDecay/Mul-op64,kernel_graph0,{22859},22859,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22859,22867,0x12c19f418c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22553,692_Default/optimizer-FP32StateAdamWeightDecay/Sub-op12,Default/optimizer-FP32StateAdamWeightDecay/Sub-op12,kernel_graph0,{22865},22865,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22868,22874,0x12c22361ba00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22559,696_Default/optimizer-FP32StateAdamWeightDecay/Mul-op65,Default/optimizer-FP32StateAdamWeightDecay/Mul-op65,kernel_graph0,{22872},22872,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22870,22878,0x12c204812e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22560,697_Default/optimizer-FP32StateAdamWeightDecay/Mul-op67,Default/optimizer-FP32StateAdamWeightDecay/Mul-op67,kernel_graph0,{22876},22876,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22872,22897,0x12c19f418c00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22561,698_Default/learning_rate-CosineWithWarmUpLR/Add-op47,Default/learning_rate-CosineWithWarmUpLR/Add-op47,kernel_graph0,{22885-22895},22895,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22876,22895,0x12c22361ba00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22562,699_Default/learning_rate-CosineWithWarmUpLR/Add-op48,Default/learning_rate-CosineWithWarmUpLR/Add-op48,kernel_graph0,{22880-22893},22893,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22880,22884,0x12c22e21fe00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22562,700_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op13,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op13,kernel_graph0,{22882},22882,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22882,22887,0x12c204812e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22563,701_Default/learning_rate-CosineWithWarmUpLR/Add-op49,Default/learning_rate-CosineWithWarmUpLR/Add-op49,kernel_graph0,{22885},22885,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22885,22890,0x12c22e21fe00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22564,702_Default/optimizer-FP32StateAdamWeightDecay/Div-op13,Default/optimizer-FP32StateAdamWeightDecay/Div-op13,kernel_graph0,{22888},22888,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22888,22893,0x12c204812e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22565,703_Default/optimizer-FP32StateAdamWeightDecay/Mul-op69,Default/optimizer-FP32StateAdamWeightDecay/Mul-op69,kernel_graph0,{22891},22891,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22891,22899,0x12c22e21fe00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22566,704_Default/optimizer-FP32StateAdamWeightDecay/Sub-op13,Default/optimizer-FP32StateAdamWeightDecay/Sub-op13,kernel_graph0,{22897},22897,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22900,22906,0x12c22361ba00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22570,708_Default/optimizer-FP32StateAdamWeightDecay/Mul-op70,Default/optimizer-FP32StateAdamWeightDecay/Mul-op70,kernel_graph0,{22904},22904,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22902,22910,0x12c22e21fe00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22571,709_Default/optimizer-FP32StateAdamWeightDecay/Mul-op72,Default/optimizer-FP32StateAdamWeightDecay/Mul-op72,kernel_graph0,{22908},22908,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22904,22929,0x12c204812e00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22572,710_Default/learning_rate-CosineWithWarmUpLR/Add-op50,Default/learning_rate-CosineWithWarmUpLR/Add-op50,kernel_graph0,{22917-22927},22927,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22908,22927,0x12c22361ba00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22573,711_Default/learning_rate-CosineWithWarmUpLR/Add-op51,Default/learning_rate-CosineWithWarmUpLR/Add-op51,kernel_graph0,{22912-22925},22925,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22912,22916,0x12c1f1c12800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22573,712_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op14,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op14,kernel_graph0,{22914},22914,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22914,22919,0x12c22e21fe00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22574,713_Default/learning_rate-CosineWithWarmUpLR/Add-op52,Default/learning_rate-CosineWithWarmUpLR/Add-op52,kernel_graph0,{22917},22917,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22917,22922,0x12c1f1c12800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22574,714_Default/optimizer-FP32StateAdamWeightDecay/Div-op14,Default/optimizer-FP32StateAdamWeightDecay/Div-op14,kernel_graph0,{22920},22920,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22920,22925,0x12c22e21fe00,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22574,715_Default/optimizer-FP32StateAdamWeightDecay/Mul-op74,Default/optimizer-FP32StateAdamWeightDecay/Mul-op74,kernel_graph0,{22923},22923,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22923,22931,0x12c1f1c12800,0,DefaultEnhancedAscendMemoryPool,180355584,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22575,716_Default/optimizer-FP32StateAdamWeightDecay/Sub-op14,Default/optimizer-FP32StateAdamWeightDecay/Sub-op14,kernel_graph0,{22929},22929,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22932,22938,0x12c1f1c12800,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22577,720_Default/optimizer-FP32StateAdamWeightDecay/Mul-op75,Default/optimizer-FP32StateAdamWeightDecay/Mul-op75,kernel_graph0,{22936},22936,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22934,22942,0x12c19f418c00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22578,721_Default/optimizer-FP32StateAdamWeightDecay/Mul-op77,Default/optimizer-FP32StateAdamWeightDecay/Mul-op77,kernel_graph0,{22940},22940,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22936,22961,0x12c1be818e00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22578,722_Default/learning_rate-CosineWithWarmUpLR/Add-op53,Default/learning_rate-CosineWithWarmUpLR/Add-op53,kernel_graph0,{22949-22959},22959,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22940,22959,0x12c1f1c12800,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22578,723_Default/learning_rate-CosineWithWarmUpLR/Add-op54,Default/learning_rate-CosineWithWarmUpLR/Add-op54,kernel_graph0,{22944-22957},22957,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22944,22948,0x12c180018a00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22579,724_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op15,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op15,kernel_graph0,{22946},22946,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22946,22951,0x12c19f418c00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22580,725_Default/learning_rate-CosineWithWarmUpLR/Add-op55,Default/learning_rate-CosineWithWarmUpLR/Add-op55,kernel_graph0,{22949},22949,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22949,22954,0x12c180018a00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22581,726_Default/optimizer-FP32StateAdamWeightDecay/Div-op15,Default/optimizer-FP32StateAdamWeightDecay/Div-op15,kernel_graph0,{22952},22952,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22952,22957,0x12c19f418c00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22581,727_Default/optimizer-FP32StateAdamWeightDecay/Mul-op79,Default/optimizer-FP32StateAdamWeightDecay/Mul-op79,kernel_graph0,{22955},22955,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22955,22963,0x12c180018a00,0,DefaultEnhancedAscendMemoryPool,524288512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22582,728_Default/optimizer-FP32StateAdamWeightDecay/Sub-op15,Default/optimizer-FP32StateAdamWeightDecay/Sub-op15,kernel_graph0,{22961},22961,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22964,22970,0x12c22361ba00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22585,732_Default/optimizer-FP32StateAdamWeightDecay/Mul-op80,Default/optimizer-FP32StateAdamWeightDecay/Mul-op80,kernel_graph0,{22968},22968,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22966,22974,0x12c22361fc00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22587,733_Default/optimizer-FP32StateAdamWeightDecay/Mul-op82,Default/optimizer-FP32StateAdamWeightDecay/Mul-op82,kernel_graph0,{22972},22972,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22968,22993,0x12c223623e00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22588,734_Default/learning_rate-CosineWithWarmUpLR/Add-op56,Default/learning_rate-CosineWithWarmUpLR/Add-op56,kernel_graph0,{22981-22991},22991,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22972,22991,0x12c223617800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22589,735_Default/learning_rate-CosineWithWarmUpLR/Add-op57,Default/learning_rate-CosineWithWarmUpLR/Add-op57,kernel_graph0,{22976-22989},22989,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22976,22980,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22589,736_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op16,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op16,kernel_graph0,{22978},22978,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22978,22983,0x12c22361ba00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22592,737_Default/learning_rate-CosineWithWarmUpLR/Add-op58,Default/learning_rate-CosineWithWarmUpLR/Add-op58,kernel_graph0,{22981},22981,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22981,22986,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22593,738_Default/optimizer-FP32StateAdamWeightDecay/Div-op16,Default/optimizer-FP32StateAdamWeightDecay/Div-op16,kernel_graph0,{22984},22984,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22984,22989,0x12c22361ba00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22593,739_Default/optimizer-FP32StateAdamWeightDecay/Mul-op84,Default/optimizer-FP32StateAdamWeightDecay/Mul-op84,kernel_graph0,{22987},22987,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22987,22995,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22594,740_Default/optimizer-FP32StateAdamWeightDecay/Sub-op16,Default/optimizer-FP32StateAdamWeightDecay/Sub-op16,kernel_graph0,{22993},22993,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22996,23002,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22598,744_Default/optimizer-FP32StateAdamWeightDecay/Mul-op85,Default/optimizer-FP32StateAdamWeightDecay/Mul-op85,kernel_graph0,{23000},23000,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 22998,23006,0x12c223617800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22600,745_Default/optimizer-FP32StateAdamWeightDecay/Mul-op87,Default/optimizer-FP32StateAdamWeightDecay/Mul-op87,kernel_graph0,{23004},23004,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 23000,23025,0x12c22361ba00,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22600,746_Default/learning_rate-CosineWithWarmUpLR/Add-op59,Default/learning_rate-CosineWithWarmUpLR/Add-op59,kernel_graph0,{23013-23023},23023,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 23004,23023,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22601,747_Default/learning_rate-CosineWithWarmUpLR/Add-op60,Default/learning_rate-CosineWithWarmUpLR/Add-op60,kernel_graph0,{23008-23021},23021,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 23008,23012,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22602,748_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op17,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op17,kernel_graph0,{23010},23010,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 23010,23015,0x12c223613600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22603,749_Default/learning_rate-CosineWithWarmUpLR/Add-op61,Default/learning_rate-CosineWithWarmUpLR/Add-op61,kernel_graph0,{23013},23013,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 23013,23018,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22604,750_Default/optimizer-FP32StateAdamWeightDecay/Div-op17,Default/optimizer-FP32StateAdamWeightDecay/Div-op17,kernel_graph0,{23016},23016,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 23016,23021,0x12c223613600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22605,751_Default/optimizer-FP32StateAdamWeightDecay/Mul-op89,Default/optimizer-FP32StateAdamWeightDecay/Mul-op89,kernel_graph0,{23019},23019,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 23019,23027,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22606,752_Default/optimizer-FP32StateAdamWeightDecay/Sub-op17,Default/optimizer-FP32StateAdamWeightDecay/Sub-op17,kernel_graph0,{23025},23025,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 23028,23034,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22608,756_Default/optimizer-FP32StateAdamWeightDecay/Mul-op90,Default/optimizer-FP32StateAdamWeightDecay/Mul-op90,kernel_graph0,{23032},23032,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 23030,23038,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22608,757_Default/optimizer-FP32StateAdamWeightDecay/Mul-op92,Default/optimizer-FP32StateAdamWeightDecay/Mul-op92,kernel_graph0,{23036},23036,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 23032,23057,0x12c223613600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22608,758_Default/learning_rate-CosineWithWarmUpLR/Add-op62,Default/learning_rate-CosineWithWarmUpLR/Add-op62,kernel_graph0,{23045-23055},23055,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 23036,23055,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22609,759_Default/learning_rate-CosineWithWarmUpLR/Add-op63,Default/learning_rate-CosineWithWarmUpLR/Add-op63,kernel_graph0,{23040-23053},23053,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 23040,23044,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22609,760_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op18,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op18,kernel_graph0,{23042},23042,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 23042,23047,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22610,761_Default/learning_rate-CosineWithWarmUpLR/Add-op64,Default/learning_rate-CosineWithWarmUpLR/Add-op64,kernel_graph0,{23045},23045,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 23045,23050,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22610,762_Default/optimizer-FP32StateAdamWeightDecay/Div-op18,Default/optimizer-FP32StateAdamWeightDecay/Div-op18,kernel_graph0,{23048},23048,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 23048,23053,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22610,763_Default/optimizer-FP32StateAdamWeightDecay/Mul-op94,Default/optimizer-FP32StateAdamWeightDecay/Mul-op94,kernel_graph0,{23051},23051,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 23051,23059,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22612,764_Default/optimizer-FP32StateAdamWeightDecay/Sub-op18,Default/optimizer-FP32StateAdamWeightDecay/Sub-op18,kernel_graph0,{23057},23057,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 23060,23066,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22615,768_Default/optimizer-FP32StateAdamWeightDecay/Mul-op97,Default/optimizer-FP32StateAdamWeightDecay/Mul-op97,kernel_graph0,{23064},23064,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 23062,23070,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22615,769_Default/optimizer-FP32StateAdamWeightDecay/Mul-op95,Default/optimizer-FP32StateAdamWeightDecay/Mul-op95,kernel_graph0,{23068},23068,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 23064,23087,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22616,770_Default/learning_rate-CosineWithWarmUpLR/Add-op66,Default/learning_rate-CosineWithWarmUpLR/Add-op66,kernel_graph0,{23072-23085},23085,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 23068,23089,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22617,771_Default/learning_rate-CosineWithWarmUpLR/Add-op65,Default/learning_rate-CosineWithWarmUpLR/Add-op65,kernel_graph0,{23077-23087},23087,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 23072,23076,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22617,772_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op19,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op19,kernel_graph0,{23074},23074,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 23074,23079,0x12c180014800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22620,773_Default/learning_rate-CosineWithWarmUpLR/Add-op67,Default/learning_rate-CosineWithWarmUpLR/Add-op67,kernel_graph0,{23077},23077,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 23077,23082,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22621,774_Default/optimizer-FP32StateAdamWeightDecay/Div-op19,Default/optimizer-FP32StateAdamWeightDecay/Div-op19,kernel_graph0,{23080},23080,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 23080,23085,0x12c180014800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22621,775_Default/optimizer-FP32StateAdamWeightDecay/Mul-op99,Default/optimizer-FP32StateAdamWeightDecay/Mul-op99,kernel_graph0,{23083},23083,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 23083,23091,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22624,776_Default/optimizer-FP32StateAdamWeightDecay/Sub-op19,Default/optimizer-FP32StateAdamWeightDecay/Sub-op19,kernel_graph0,{23089},23089,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 23092,23098,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22627,780_Default/optimizer-FP32StateAdamWeightDecay/Mul-op102,Default/optimizer-FP32StateAdamWeightDecay/Mul-op102,kernel_graph0,{23096},23096,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 23094,23102,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22627,781_Default/optimizer-FP32StateAdamWeightDecay/Mul-op100,Default/optimizer-FP32StateAdamWeightDecay/Mul-op100,kernel_graph0,{23100},23100,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 23096,23124,0x12c180004800,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22629,782_Default/learning_rate-CosineWithWarmUpLR/Add-op69,Default/learning_rate-CosineWithWarmUpLR/Add-op69,kernel_graph0,{23104-23122},23122,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 23100,23122,0x12c18000c400,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22630,783_Default/learning_rate-CosineWithWarmUpLR/Add-op68,Default/learning_rate-CosineWithWarmUpLR/Add-op68,kernel_graph0,{23109-23120},23120,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 23104,23108,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22630,784_Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op20,Default/optimizer-FP32StateAdamWeightDecay/Sqrt-op20,kernel_graph0,{23106},23106,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 23106,23111,0x12c180010600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22632,785_Default/learning_rate-CosineWithWarmUpLR/Add-op70,Default/learning_rate-CosineWithWarmUpLR/Add-op70,kernel_graph0,{23109},23109,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 23109,23115,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22633,786_Default/optimizer-FP32StateAdamWeightDecay/Div-op20,Default/optimizer-FP32StateAdamWeightDecay/Div-op20,kernel_graph0,{23112},23112,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 23112,23118,0x12c180010600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22633,787_Default/optimizer-FP32StateAdamWeightDecay/Mul-op104,Default/optimizer-FP32StateAdamWeightDecay/Mul-op104,kernel_graph0,{23116},23116,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 23116,23120,0x12c180000600,0,DefaultEnhancedAscendMemoryPool,16896,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22635,788_Default/optimizer-FP32StateAdamWeightDecay/Sub-op20,Default/optimizer-FP32StateAdamWeightDecay/Sub-op20,kernel_graph0,{23118},23118,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 23125,9223372036854775807,0x12c180008c00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/memory_manager_actor.cc,48,Kernel,22638,792_Default/TensorMove-op2,Default/TensorMove-op2,kernel_graph0,{},,File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/traceback.py;Line:211;Function:extract_stack|,0,0, 23129,9223372036854775807,0x12c75d35ee00,0,DefaultEnhancedAscendMemoryPool,512,16181624832,mindspore/ccsrc/runtime/core/actors/base/output_actor.cc,689,Other,23127,kernel_graph_0_OutputActor,UpdateOutputDeviceAddress,,{},,File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:141;Function:|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:139;Function:run_llama|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/memory/test_llama_1p.py;Line:103;Function:run_llama_1p_no_somas_grad_accu|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/_checkparam.py;Line:1398;Function:wrapper|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/trainer.py;Line:478;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/causal_language_modeling/causal_language_modeling.py;Line:114;Function:train|File:/home/jenkins/mindspore/testcases/testcases/tests/st/networks/mindformers/mindformers/trainer/base_trainer.py;Line:1238;Function:training_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1349;Function:train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:302;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:121;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1034;Function:_train|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/train/model.py;Line:1215;Function:_train_process|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1358;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/nn/cell.py;Line:1748;Function:compile_and_run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2181;Function:__call__|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2232;Function:run|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:187;Function:wrapper|File:/home/jenkins/anaconda3/envs/ci39/lib/python3.9/site-packages/mindspore/common/api.py;Line:2212;Function:_exec_pip|,1,0,