<!-- BaNnErBlUrFlE-BoDy-start -->
<!-- Preheader Text : BEGIN -->
<div style="display:none !important;display:none;visibility:hidden;mso-hide:all;font-size:1px;color:#ffffff;line-height:1px;height:0px;max-height:0px;opacity:0;overflow:hidden;">
Hi Barry, sorry for my last results. I didn’t fully understand the stage profiling and logging in PETSc, now I only record KSPSolve() stage of my program. Some sample codes are as follow, // Static variable to keep track of the stage counter
</div>
<!-- Preheader Text : END -->
<!-- Email Banner : BEGIN -->
<div style="display:none !important;display:none;visibility:hidden;mso-hide:all;font-size:1px;color:#ffffff;line-height:1px;height:0px;max-height:0px;opacity:0;overflow:hidden;">ZjQcmQRYFpfptBannerStart</div>
<!--[if ((ie)|(mso))]>
<table border="0" cellspacing="0" cellpadding="0" width="100%" style="padding: 16px 0px 16px 0px; direction: ltr" ><tr><td>
<table border="0" cellspacing="0" cellpadding="0" style="padding: 0px 10px 5px 6px; width: 100%; border-radius:4px; border-top:4px solid #90a4ae;background-color:#D0D8DC;"><tr><td valign="top">
<table align="left" border="0" cellspacing="0" cellpadding="0" style="padding: 4px 8px 4px 8px">
<tr><td style="color:#000000; font-family: 'Arial', sans-serif; font-weight:bold; font-size:14px; direction: ltr">
This Message Is From an External Sender
</td></tr>
<tr><td style="color:#000000; font-weight:normal; font-family: 'Arial', sans-serif; font-size:12px; direction: ltr">
This message came from outside your organization.
</td></tr>
</table>
</td></tr></table>
</td></tr></table>
<![endif]-->
<![if !((ie)|(mso))]>
<div dir="ltr" id="pfptBannersqiow23" style="all: revert !important; display:block !important; text-align: left !important; margin:16px 0px 16px 0px !important; padding:8px 16px 8px 16px !important; border-radius: 4px !important; min-width: 200px !important; background-color: #D0D8DC !important; background-color: #D0D8DC; border-top: 4px solid #90a4ae !important; border-top: 4px solid #90a4ae;">
<div id="pfptBannersqiow23" style="all: unset !important; float:left !important; display:block !important; margin: 0px 0px 1px 0px !important; max-width: 600px !important;">
<div id="pfptBannersqiow23" style="all: unset !important; display:block !important; visibility: visible !important; background-color: #D0D8DC !important; color:#000000 !important; color:#000000; font-family: 'Arial', sans-serif !important; font-family: 'Arial', sans-serif; font-weight:bold !important; font-weight:bold; font-size:14px !important; line-height:18px !important; line-height:18px">
This Message Is From an External Sender
</div>
<div id="pfptBannersqiow23" style="all: unset !important; display:block !important; visibility: visible !important; background-color: #D0D8DC !important; color:#000000 !important; color:#000000; font-weight:normal; font-family: 'Arial', sans-serif !important; font-family: 'Arial', sans-serif; font-size:12px !important; line-height:18px !important; line-height:18px; margin-top:2px !important;">
This message came from outside your organization.
</div>
</div>
<div style="clear: both !important; display: block !important; visibility: hidden !important; line-height: 0 !important; font-size: 0.01px !important; height: 0px"> </div>
</div>
<![endif]>
<div style="display:none !important;display:none;visibility:hidden;mso-hide:all;font-size:1px;color:#ffffff;line-height:1px;height:0px;max-height:0px;opacity:0;overflow:hidden;">ZjQcmQRYFpfptBannerEnd</div>
<!-- Email Banner : END -->
<!-- BaNnErBlUrFlE-BoDy-end -->
<html xmlns:o="urn:schemas-microsoft-com:office:office" xmlns:w="urn:schemas-microsoft-com:office:word" xmlns:m="http://schemas.microsoft.com/office/2004/12/omml" xmlns="http://www.w3.org/TR/REC-html40">
<head><!-- BaNnErBlUrFlE-HeAdEr-start -->
<style>
#pfptBannersqiow23 { all: revert !important; display: block !important;
visibility: visible !important; opacity: 1 !important;
background-color: #D0D8DC !important;
max-width: none !important; max-height: none !important }
.pfptPrimaryButtonsqiow23:hover, .pfptPrimaryButtonsqiow23:focus {
background-color: #b4c1c7 !important; }
.pfptPrimaryButtonsqiow23:active {
background-color: #90a4ae !important; }
</style>
<!-- BaNnErBlUrFlE-HeAdEr-end -->
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
<meta name="Generator" content="Microsoft Word 15 (filtered medium)">
<style><!--
/* Font Definitions */
@font-face
{font-family:"Cambria Math";
panose-1:2 4 5 3 5 4 6 3 2 4;}
@font-face
{font-family:DengXian;
panose-1:2 1 6 0 3 1 1 1 1 1;}
@font-face
{font-family:Calibri;
panose-1:2 15 5 2 2 2 4 3 2 4;}
@font-face
{font-family:Aptos;
panose-1:2 11 0 4 2 2 2 2 2 4;}
@font-face
{font-family:"\@DengXian";
panose-1:2 1 6 0 3 1 1 1 1 1;}
@font-face
{font-family:"Segoe UI";
panose-1:2 11 5 2 4 2 4 2 2 3;}
/* Style Definitions */
p.MsoNormal, li.MsoNormal, div.MsoNormal
{margin:0cm;
font-size:12.0pt;
font-family:"Aptos",sans-serif;}
a:link, span.MsoHyperlink
{mso-style-priority:99;
color:blue;
text-decoration:underline;}
span.apple-converted-space
{mso-style-name:apple-converted-space;}
span.gmailsignatureprefix
{mso-style-name:gmailsignatureprefix;}
.MsoChpDefault
{mso-style-type:export-only;
font-size:10.0pt;
mso-ligatures:none;}
@page WordSection1
{size:612.0pt 792.0pt;
margin:72.0pt 72.0pt 72.0pt 72.0pt;}
div.WordSection1
{page:WordSection1;}
/* List Definitions */
@list l0
{mso-list-id:521433824;
mso-list-template-ids:2038466866;}
@list l0:level1
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:36.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l0:level2
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:72.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l0:level3
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:108.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l0:level4
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:144.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l0:level5
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:180.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l0:level6
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:216.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l0:level7
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:252.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l0:level8
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:288.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l0:level9
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:324.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l1
{mso-list-id:1512989947;
mso-list-template-ids:-1597618000;}
@list l1:level1
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:36.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l1:level2
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:72.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l1:level3
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:108.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l1:level4
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:144.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l1:level5
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:180.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l1:level6
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:216.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l1:level7
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:252.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l1:level8
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:288.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l1:level9
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:324.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
ol
{margin-bottom:0cm;}
ul
{margin-bottom:0cm;}
--></style>
</head>
<body lang="en-CN" link="blue" vlink="purple" style="word-wrap:break-word;line-break:after-white-space">
<div class="WordSection1">
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt">Hi Barry, sorry for my last results. I didn’t fully understand the stage profiling and logging in PETSc, now I only record KSPSolve() stage of my program. Some sample codes are as follow,<br>
<br>
// Static variable to keep track of the stage counter<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt"> static int stageCounter = 1;<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt"><o:p> </o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt"> // Generate a unique stage name<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt"> std::ostringstream oss;<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt"> oss << "Stage " << stageCounter << " of Code";<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt"> std::string stageName = oss.str();<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt"><o:p> </o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt"> // Register the stage<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt"> PetscLogStage stagenum;<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt"><o:p> </o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt"> PetscLogStageRegister(stageName.c_str(), &stagenum);<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt"> PetscLogStagePush(stagenum);<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt"><o:p> </o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt"> <b>
KSPSolve(*ksp_ptr, b, x);<o:p></o:p></b></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt"><o:p> </o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt"> PetscLogStagePop();<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt"> stageCounter++;
<br>
<br>
I have attached my new logging results, there are 1 main stage and 4 other stages where each one is KSPSolve() call.
<br>
<br>
To provide some additional backgrounds, if you recall, I have been trying to get efficient iterative solution using multithreading. I found out by compiling PETSc with Intel MKL library instead of OpenBLAS, I am able to perform sparse matrix-vector multiplication
faster, I am using MATSEQAIJMKL. This makes the shell matrix vector product in each iteration scale well with the #of threads. However, I found out the total GMERS solve time (~KSPSolve() time) is not scaling well the #of threads.<br>
<br>
>From the logging results I learned that when performing KSPSolve(), there are some CPU overheads in PCApply() and KSPGMERSOrthog(). I ran my programs using different number of threads and plotted the time consumption for PCApply() and KSPGMERSOrthog() against
#of thread. I found out these two operations are not scaling with the threads at all! My results are attached as the pdf to give you a clear view.<br>
<br>
My questions is,<br>
<br>
>From my understanding, in PCApply, MatSolve() is involved, KSPGMERSOrthog() will have many vector operations, so why these two parts can’t scale well with the # of threads when the intel MKL library is linked?<br>
<br>
Thank you,<br>
Yongzhong<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt"><o:p> </o:p></span></p>
<div id="mail-editor-reference-message-container">
<div>
<div style="border:none;border-top:solid #B5C4DF 1.0pt;padding:3.0pt 0cm 0cm 0cm">
<p class="MsoNormal" style="margin-bottom:12.0pt"><b><span style="color:black">From:
</span></b><span style="color:black">Barry Smith <bsmith@petsc.dev><br>
<b>Date: </b>Friday, June 14, 2024 at 11:36</span><span style="font-family:"Arial",sans-serif;color:black"> </span><span style="color:black">AM<br>
<b>To: </b>Yongzhong Li <yongzhong.li@mail.utoronto.ca><br>
<b>Cc: </b>petsc-users@mcs.anl.gov <petsc-users@mcs.anl.gov>, petsc-maint@mcs.anl.gov <petsc-maint@mcs.anl.gov>, Piero Triverio <piero.triverio@utoronto.ca><br>
<b>Subject: </b>Re: [petsc-maint] Assistance Needed with PETSc KSPSolve Performance Issue<o:p></o:p></span></p>
</div>
<div>
<p class="MsoNormal"><o:p> </o:p></p>
</div>
<p class="MsoNormal"> I am a bit confused. Without the initial guess computation, there are still a bunch of events I don't understand </p>
<div>
<p class="MsoNormal"><o:p> </o:p></p>
</div>
<div>
<div>
<p class="MsoNormal">MatTranspose 79 1.0 4.0598e+01 1.0 0.00e+00 0.0 0.0e+00 0.0e+00 0.0e+00 0 0 0 0 0 0 0 0 0 0 0</p>
</div>
<div>
<p class="MsoNormal">MatMatMultSym 110 1.0 1.7419e+02 1.0 0.00e+00 0.0 0.0e+00 0.0e+00 0.0e+00 1 0 0 0 0 1 0 0 0 0 0</p>
</div>
<div>
<p class="MsoNormal">MatMatMultNum 90 1.0 1.2640e+02 1.0 0.00e+00 0.0 0.0e+00 0.0e+00 0.0e+00 1 0 0 0 0 1 0 0 0 0 0</p>
</div>
<div>
<p class="MsoNormal">MatMatMatMultSym 20 1.0 1.3049e+02 1.0 0.00e+00 0.0 0.0e+00 0.0e+00 0.0e+00 1 0 0 0 0 1 0 0 0 0 0</p>
</div>
<div>
<p class="MsoNormal">MatRARtSym 25 1.0 1.2492e+02 1.0 0.00e+00 0.0 0.0e+00 0.0e+00 0.0e+00 1 0 0 0 0 1 0 0 0 0 0</p>
</div>
<div>
<p class="MsoNormal">MatMatTrnMultSym 25 1.0 8.8265e+01 1.0 0.00e+00 0.0 0.0e+00 0.0e+00 0.0e+00 0 0 0 0 0 0 0 0 0 0 0</p>
</div>
<div>
<p class="MsoNormal">MatMatTrnMultNum 25 1.0 2.4820e+02 1.0 6.83e+10 1.0 0.0e+00 0.0e+00 0.0e+00 1 0 0 0 0 1 0 0 0 0 275</p>
</div>
<div>
<p class="MsoNormal">MatTrnMatMultSym 10 1.0 7.2984e-01 1.0 0.00e+00 0.0 0.0e+00 0.0e+00 0.0e+00 0 0 0 0 0 0 0 0 0 0 0</p>
</div>
<div>
<p class="MsoNormal">MatTrnMatMultNum 10 1.0 9.3128e-01 1.0 0.00e+00 0.0 0.0e+00 0.0e+00 0.0e+00 0 0 0 0 0 0 0 0 0 0 0</p>
</div>
<div>
<p class="MsoNormal"><o:p> </o:p></p>
</div>
<div>
<p class="MsoNormal">in addition there are many more VecMAXPY then VecMDot (in GMRES they are each done the same number of times)</p>
</div>
<div>
<p class="MsoNormal"><o:p> </o:p></p>
</div>
<div>
<div>
<p class="MsoNormal">VecMDot 5588 1.0 1.7183e+03 1.0 2.06e+13 1.0 0.0e+00 0.0e+00 0.0e+00 8 10 0 0 0 8 10 0 0 0 12016</p>
</div>
</div>
<div>
<div>
<p class="MsoNormal">VecMAXPY 22412 1.0 8.4898e+03 1.0 4.17e+13 1.0 0.0e+00 0.0e+00 0.0e+00 39 20 0 0 0 39 20 0 0 0 4913</p>
</div>
</div>
<div>
<p class="MsoNormal"><o:p> </o:p></p>
</div>
<div>
<p class="MsoNormal">Finally there are a huge number of </p>
</div>
<div>
<p class="MsoNormal"><o:p> </o:p></p>
</div>
<div>
<div>
<p class="MsoNormal">MatMultAdd 258048 1.0 1.4178e+03 1.0 6.10e+13 1.0 0.0e+00 0.0e+00 0.0e+00 7 29 0 0 0 7 29 0 0 0 43025</p>
</div>
</div>
<div>
<p class="MsoNormal"><o:p> </o:p></p>
</div>
<div>
<p class="MsoNormal">Are you making calls to all these routines? Are you doing this inside your MatMult() or before you call KSPSolve?</p>
</div>
<div>
<p class="MsoNormal"><o:p> </o:p></p>
</div>
<div>
<p class="MsoNormal">The reason I wanted you to make a simpler run without the initial guess code is that your events are far more complicated than would be produced by GMRES alone so it is not possible to understand the behavior you are seeing without fully
understanding all the events happening in the code.</p>
</div>
<div>
<p class="MsoNormal"><o:p> </o:p></p>
</div>
<div>
<p class="MsoNormal"> Barry</p>
</div>
<div>
<p class="MsoNormal"><o:p> </o:p></p>
</div>
<div>
<p class="MsoNormal"><br>
<br>
</p>
<blockquote style="margin-top:5.0pt;margin-bottom:5.0pt">
<div>
<p class="MsoNormal">On Jun 14, 2024, at 1:19<span style="font-family:"Arial",sans-serif"> </span>AM, Yongzhong Li <yongzhong.li@mail.utoronto.ca> wrote:</p>
</div>
<p class="MsoNormal"><o:p> </o:p></p>
<div>
<div>
<div>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt">Thanks, I have attached the results without using any KSPGuess. At low frequency, the iteration steps are quite close to the one with KSPGuess, specifically<span class="apple-converted-space"> </span><br>
<br>
KSPGuess Object: 1 MPI process</span></p>
</div>
<div>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt"> type: fischer</span></p>
</div>
<div>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt"> Model 1, size 200<br>
<br>
However, I found at higher frequency, the # of iteration steps are significant higher than the one with KSPGuess, I have attahced both of the results for your reference.<br>
<br>
Moreover, could I ask why the one without the KSPGuess options can be used for a baseline comparsion? What are we comparing here? How does it relate to the performance issue/bottleneck I found? “</span><b>I have noticed that the time taken by </b><b><span style="font-family:"Courier New"">KSPSolve</span> is<span class="apple-converted-space"> </span></b><b><span lang="EN-US">almost
two times<span class="apple-converted-space"> </span></span>greater than the CPU time for matrix-vector product multiplied by the number of iteration</b><span lang="EN-US" style="font-size:11.0pt">”<span class="apple-converted-space"> </span><br>
<br>
Thank you!<br>
Yongzhong</span></p>
</div>
<div>
<p class="MsoNormal"><span style="font-size:11.0pt"> </span></p>
</div>
<div id="mail-editor-reference-message-container">
<div>
<div style="border:none;border-top:solid windowtext 1.0pt;padding:3.0pt 0cm 0cm 0cm;border-color:currentcolor currentcolor">
<p class="MsoNormal" style="margin-bottom:12.0pt"><b>From:<span class="apple-converted-space"> </span></b>Barry Smith <<a href="mailto:bsmith@petsc.dev">bsmith@petsc.dev</a>><br>
<b>Date:<span class="apple-converted-space"> </span></b>Thursday, June 13, 2024 at 2:14<span style="font-family:"Arial",sans-serif"> </span>PM<br>
<b>To:<span class="apple-converted-space"> </span></b>Yongzhong Li <<a href="mailto:yongzhong.li@mail.utoronto.ca">yongzhong.li@mail.utoronto.ca</a>><br>
<b>Cc:<span class="apple-converted-space"> </span></b><a href="mailto:petsc-users@mcs.anl.gov">petsc-users@mcs.anl.gov</a><span class="apple-converted-space"> </span><<a href="mailto:petsc-users@mcs.anl.gov">petsc-users@mcs.anl.gov</a>>,<span class="apple-converted-space"> </span><a href="mailto:petsc-maint@mcs.anl.gov">petsc-maint@mcs.anl.gov</a><span class="apple-converted-space"> </span><<a href="mailto:petsc-maint@mcs.anl.gov">petsc-maint@mcs.anl.gov</a>>,
Piero Triverio <<a href="mailto:piero.triverio@utoronto.ca">piero.triverio@utoronto.ca</a>><br>
<b>Subject:<span class="apple-converted-space"> </span></b>Re: [petsc-maint] Assistance Needed with PETSc KSPSolve Performance Issue</p>
</div>
<div>
<div>
<p class="MsoNormal"> <o:p></o:p></p>
</div>
</div>
<div>
<p class="MsoNormal"> Can you please run the same thing without the KSPGuess option(s) for a baseline comparison?</p>
</div>
<div>
<div>
<p class="MsoNormal"> </p>
</div>
</div>
<div>
<div>
<p class="MsoNormal"> Thanks</p>
</div>
</div>
<div>
<div>
<p class="MsoNormal"> </p>
</div>
</div>
<div>
<div>
<p class="MsoNormal"> Barry</p>
</div>
</div>
<div>
<div>
<div>
<p class="MsoNormal" style="margin-bottom:12.0pt"><o:p> </o:p></p>
</div>
<blockquote style="margin-top:5.0pt;margin-bottom:5.0pt">
<div>
<div>
<p class="MsoNormal">On Jun 13, 2024, at 1:27<span style="font-family:"Arial",sans-serif"> </span>PM, Yongzhong Li <<a href="mailto:yongzhong.li@mail.utoronto.ca">yongzhong.li@mail.utoronto.ca</a>> wrote:</p>
</div>
</div>
<div>
<p class="MsoNormal"> </p>
</div>
<div>
<div id="pfptBannerxs2204y">
<div id="pfptBannerxs2204y">
<div id="pfptBannerxs2204y">
<div>
<p class="MsoNormal"><span style="font-family:"Arial",sans-serif">This Message Is From an External Sender</span></p>
</div>
</div>
<div id="pfptBannerxs2204y">
<div>
<p class="MsoNormal"><span style="font-family:"Arial",sans-serif">This message came from outside your organization.</span></p>
</div>
</div>
</div>
</div>
<div>
<div>
<div>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt">Hi Matt,<br>
<br>
I have rerun the program with the keys you provided. The system output when performing ksp solve and the final petsc log output were stored in a .txt file attached for your reference.<br>
<br>
Thanks!<br>
Yongzhong</span></p>
</div>
</div>
<div>
<div>
<p class="MsoNormal"><span style="font-size:11.0pt"> </span></p>
</div>
</div>
<div id="mail-editor-reference-message-container">
<div>
<div style="border:none;border-top:solid windowtext 1.0pt;padding:3.0pt 0cm 0cm 0cm;border-color:currentcolor">
<p class="MsoNormal" style="margin-bottom:12.0pt"><b>From:<span class="apple-converted-space"> </span></b>Matthew Knepley <<a href="mailto:knepley@gmail.com">knepley@gmail.com</a>><br>
<b>Date:<span class="apple-converted-space"> </span></b>Wednesday, June 12, 2024 at 6:46<span style="font-family:"Arial",sans-serif"> </span>PM<br>
<b>To:<span class="apple-converted-space"> </span></b>Yongzhong Li <<a href="mailto:yongzhong.li@mail.utoronto.ca">yongzhong.li@mail.utoronto.ca</a>><br>
<b>Cc:<span class="apple-converted-space"> </span></b><a href="mailto:petsc-users@mcs.anl.gov">petsc-users@mcs.anl.gov</a><span class="apple-converted-space"> </span><<a href="mailto:petsc-users@mcs.anl.gov">petsc-users@mcs.anl.gov</a>>,<span class="apple-converted-space"> </span><a href="mailto:petsc-maint@mcs.anl.gov">petsc-maint@mcs.anl.gov</a><span class="apple-converted-space"> </span><<a href="mailto:petsc-maint@mcs.anl.gov">petsc-maint@mcs.anl.gov</a>>,
Piero Triverio <<a href="mailto:piero.triverio@utoronto.ca">piero.triverio@utoronto.ca</a>><br>
<b>Subject:<span class="apple-converted-space"> </span></b>Re: [petsc-maint] Assistance Needed with PETSc KSPSolve Performance Issue<o:p></o:p></p>
</div>
<table class="MsoNormalTable" border="0" cellspacing="0" cellpadding="0" align="left" width="100%" style="width:100.0%;display:table;float:none">
<tbody>
<tr>
<td style="background:#A6A6A6;padding:5.25pt 1.5pt 5.25pt 1.5pt"></td>
<td width="100%" style="width:100.0%;background:#EAEAEA;padding:5.25pt 3.75pt 5.25pt 11.25pt">
<div>
<div>
<div>
<p class="MsoNormal" style="mso-element:frame;mso-element-frame-hspace:2.25pt;mso-element-wrap:around;mso-element-anchor-vertical:paragraph;mso-element-anchor-horizontal:column;mso-height-rule:exactly">
<span lang="ZH-CN" style="font-size:9.0pt;font-family:DengXian;color:#212121">你通常不会收到来自</span><span class="apple-converted-space"><span style="font-size:9.0pt;font-family:"Segoe UI",sans-serif;color:#212121"> </span></span><span style="color:black"><a href="mailto:knepley@gmail.com"><span style="font-size:9.0pt;font-family:"Segoe UI",sans-serif">knepley@gmail.com</span></a></span><span class="apple-converted-space"><span style="font-size:9.0pt;font-family:"Segoe UI",sans-serif;color:#212121"> </span></span><span lang="ZH-CN" style="font-size:9.0pt;font-family:DengXian;color:#212121">的电子邮件。</span><span style="color:black"><a href="https://urldefense.us/v3/__https://aka.ms/LearnAboutSenderIdentification__;!!G_uCfscf7eWS!djGfJnEhNJROfsMsBJy5u_KoRKbug55xZ64oHKUFnH2cWku_Th1hwt4TDdoMd8pWYVDzJeqJslMNZwpO3y0Et94d31qk-oCEwo4$"><span lang="ZH-CN" style="font-size:9.0pt;font-family:DengXian">了解这一点为什么很重要</span></a></span></p>
</div>
</div>
</div>
</td>
<td width="75" style="width:56.25pt;background:#EAEAEA;padding:5.25pt 3.75pt 5.25pt 3.75pt">
</td>
</tr>
</tbody>
</table>
<div>
<div>
<div>
<div>
<div>
<p class="MsoNormal">On Wed, Jun 12, 2024 at 6:36<span style="font-family:"Arial",sans-serif"> </span>PM Yongzhong Li <<a href="mailto:yongzhong.li@mail.utoronto.ca">yongzhong.li@mail.utoronto.ca</a>> wrote:<o:p></o:p></p>
</div>
</div>
</div>
<div>
<blockquote style="border:none;border-left:solid windowtext 1.0pt;padding:0cm 0cm 0cm 6.0pt;margin-left:4.8pt;margin-top:5.0pt;margin-right:0cm;margin-bottom:5.0pt;border-color:currentcolor currentcolor currentcolor rgb(204,204,204)">
<div>
<div>
<div>
<div>
<p class="MsoNormal"><span style="font-size:1.0pt;color:white">Dear PETSc’s developers, I hope this email finds you well. I am currently working on a project using PETSc and have encountered a performance issue with the KSPSolve function. Specifically, I have
noticed that the time taken by KSPSolve is<span class="apple-converted-space"> </span></span></p>
</div>
</div>
</div>
<div>
<div>
<div>
<p class="MsoNormal"><span style="font-size:1.0pt;color:white">ZjQcmQRYFpfptBannerStart</span></p>
</div>
</div>
</div>
<div style="border:none;border-top:solid windowtext 3.0pt;padding:6.0pt 0cm 0cm 0cm;margin-top:12.0pt;margin-bottom:12.0pt;border-color:currentcolor;border-radius:4px;min-width:200px" id="m_5223539340070699115pfptBannerpeukvww">
<div style="margin-bottom:.75pt;float:left" id="m_5223539340070699115pfptBannerpeukvww">
<div id="m_5223539340070699115pfptBannerpeukvww">
<div>
<div>
<p class="MsoNormal" style="line-height:13.5pt;background:#D0D8DC"><b><span style="font-size:10.5pt;font-family:"Arial",sans-serif;color:black">This Message Is From an External Sender</span></b></p>
</div>
</div>
</div>
<div style="margin-top:1.5pt" id="m_5223539340070699115pfptBannerpeukvww">
<div>
<div>
<p class="MsoNormal" style="line-height:13.5pt;background:#D0D8DC"><span style="font-size:9.0pt;font-family:"Arial",sans-serif;color:black">This message came from outside your organization.</span></p>
</div>
</div>
</div>
</div>
<div>
<div>
<div>
<p class="MsoNormal" style="background:#D0D8DC"><span style="font-size:1.0pt;color:black"> </span></p>
</div>
</div>
</div>
</div>
<div>
<div>
<div>
<p class="MsoNormal"><span style="font-size:1.0pt;color:white">ZjQcmQRYFpfptBannerEnd</span></p>
</div>
</div>
</div>
<div>
<div>
<div>
<div>
<p class="MsoNormal">Dear<span class="apple-converted-space"> </span><span lang="EN-US">PETSc’s developers</span>,</p>
</div>
</div>
<div>
<div>
<p class="MsoNormal">I hope this email finds you well.</p>
</div>
</div>
<div>
<div>
<p class="MsoNormal">I am currently working on a project using PETSc and have encountered a performance issue with the <span style="font-family:"Courier New"">KSPSolve</span> function. Specifically,<span class="apple-converted-space"> </span><b>I have noticed
that the time taken by </b><b><span style="font-family:"Courier New"">KSPSolve</span> is<span class="apple-converted-space"> </span></b><b><span lang="EN-US">almost two times<span class="apple-converted-space"> </span></span>greater than the CPU time for matrix-vector
product multiplied by the number of iteration steps</b>.<span class="apple-converted-space"> </span><span lang="EN-US">I use C++ chrono to record CPU time.</span></p>
</div>
</div>
<div>
<div>
<p class="MsoNormal">For context, I am using a shell system matrix<span class="apple-converted-space"><span lang="EN-US"> </span></span><span lang="EN-US">A</span>. Despite my efforts to<span class="apple-converted-space"> </span><span lang="EN-US">parallelize</span><span class="apple-converted-space"> </span>the
matrix-vector product<span class="apple-converted-space"><span lang="EN-US"> </span></span><span lang="EN-US">(Ax)</span>, the overall solve time remains higher<span class="apple-converted-space"><span lang="EN-US"> </span></span><span lang="EN-US">than the
matrix vector product per iteration indicates when multiple threads were used</span>. Here are a few details of my setup:</p>
</div>
</div>
<ul style="margin-top:0cm" type="disc">
<li class="MsoNormal" style="mso-list:l0 level1 lfo3"><b>Matrix Type</b>: Shell system matrix<o:p></o:p></li><li class="MsoNormal" style="mso-list:l0 level1 lfo3"><b>Preconditioner</b>:<span class="apple-converted-space"> </span><span lang="EN-US">Shell PC</span><o:p></o:p></li><li class="MsoNormal" style="mso-list:l0 level1 lfo3"><b>Parallel Environment</b>:<span class="apple-converted-space"> </span><span lang="EN-US">Using Intel MKL as PETSc’s BLAS/LAPACK library, multithreading is enabled</span><o:p></o:p></li></ul>
<div>
<div>
<p class="MsoNormal"><span lang="EN-US">I have considered several potential reasons, such as preconditioner setup, additional solver operations, and the inherent overhead of using a shell system matrix.<span class="apple-converted-space"> </span><b>However,
since KSPSolve is a high-level API, I have been unable to pinpoint the exact cause of the increased solve time.</b></span><o:p></o:p></p>
</div>
</div>
<div>
<div>
<p class="MsoNormal"><span lang="EN-US">Have you observed the same issue?<span class="apple-converted-space"> </span></span>Could you<span class="apple-converted-space"><span lang="EN-US"> </span></span><span lang="EN-US">please<span class="apple-converted-space"> </span></span>provide
some<span class="apple-converted-space"> </span><span lang="EN-US">experience<span class="apple-converted-space"> </span></span>on how to diagnose and address this performance discrepancy? Any insights or recommendations you could offer would be greatly appreciated.<o:p></o:p></p>
</div>
</div>
</div>
</div>
</div>
</blockquote>
<div>
<div>
<div>
<p class="MsoNormal"> <o:p></o:p></p>
</div>
</div>
</div>
<div>
<div>
<div>
<p class="MsoNormal">For any performance question like this, we need to see the output of your code run with<o:p></o:p></p>
</div>
</div>
</div>
<div>
<div>
<div>
<p class="MsoNormal"> <o:p></o:p></p>
</div>
</div>
</div>
<div>
<div>
<div>
<p class="MsoNormal"> -ksp_view -ksp_monitor_true_residual -ksp_converged_reason -log_view<o:p></o:p></p>
</div>
</div>
</div>
<div>
<div>
<div>
<p class="MsoNormal"> <o:p></o:p></p>
</div>
</div>
</div>
<div>
<div>
<div>
<p class="MsoNormal"> Thanks,<o:p></o:p></p>
</div>
</div>
</div>
<div>
<div>
<div>
<p class="MsoNormal"> <o:p></o:p></p>
</div>
</div>
</div>
<div>
<div>
<div>
<p class="MsoNormal"> Matt<o:p></o:p></p>
</div>
</div>
</div>
<div>
<div>
<div>
<p class="MsoNormal"> <o:p></o:p></p>
</div>
</div>
</div>
<blockquote style="border:none;border-left:solid windowtext 1.0pt;padding:0cm 0cm 0cm 6.0pt;margin-left:4.8pt;margin-top:5.0pt;margin-right:0cm;margin-bottom:5.0pt;border-color:currentcolor currentcolor currentcolor rgb(204,204,204)">
<div>
<div>
<div>
<div>
<div>
<p class="MsoNormal">Thank you for your time and assistance.<o:p></o:p></p>
</div>
</div>
<div>
<div>
<p class="MsoNormal">Best<span class="apple-converted-space"><span lang="EN-US"> </span></span><span lang="EN-US">regards,</span><o:p></o:p></p>
</div>
</div>
<div>
<div>
<p class="MsoNormal"><span lang="EN-US">Yongzhong</span><o:p></o:p></p>
</div>
</div>
<div>
<div>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt;font-family:"Calibri",sans-serif">-----------------------------------------------------------</span><o:p></o:p></p>
</div>
</div>
<div>
<div>
<p class="MsoNormal"><b><span lang="EN-US" style="font-size:11.0pt;font-family:"Calibri",sans-serif">Yongzhong Li</span></b><o:p></o:p></p>
</div>
</div>
<div>
<div>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt;font-family:"Calibri",sans-serif">PhD student | Electromagnetics Group</span><o:p></o:p></p>
</div>
</div>
<div>
<div>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt;font-family:"Calibri",sans-serif">Department of Electrical & Computer Engineering</span><o:p></o:p></p>
</div>
</div>
<div>
<div>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt;font-family:"Calibri",sans-serif">University of Toronto</span><o:p></o:p></p>
</div>
</div>
<div>
<div>
<p class="MsoNormal"><a href="https://urldefense.us/v3/__http://www.modelics.org__;!!G_uCfscf7eWS!cuLttMJEcegaqu461Bt4QLsO4fASfLM5vjRbtyNhWJQiInbjgNwkGNdkFE1ebSbFjOUatYB0-jd2yQWMWzqkDFFjwMvNl3ZKAr8$" target="_blank"><span lang="EN-US" style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#0563C1">http://www.modelics.org</span></a><o:p></o:p></p>
</div>
</div>
<div>
<div>
<p class="MsoNormal"> <o:p></o:p></p>
</div>
</div>
</div>
</div>
</div>
</blockquote>
</div>
<div>
<div>
<p class="MsoNormal"><br clear="all">
<o:p></o:p></p>
</div>
</div>
<div>
<div>
<div>
<p class="MsoNormal"> <o:p></o:p></p>
</div>
</div>
</div>
<div>
<div>
<p class="MsoNormal"><span class="gmailsignatureprefix">--</span><span class="apple-converted-space"> </span><o:p></o:p></p>
</div>
</div>
<div>
<div>
<div>
<div>
<div>
<div>
<div>
<div>
<div>
<p class="MsoNormal">What most experimenters take for granted before they begin their experiments is infinitely more interesting than any results to which their experiments lead.<br>
-- Norbert Wiener<o:p></o:p></p>
</div>
</div>
</div>
<div>
<div>
<div>
<p class="MsoNormal"> <o:p></o:p></p>
</div>
</div>
</div>
<div>
<div>
<div>
<p class="MsoNormal"><a href="https://urldefense.us/v3/__http://www.cse.buffalo.edu/*knepley/__;fg!!G_uCfscf7eWS!djGfJnEhNJROfsMsBJy5u_KoRKbug55xZ64oHKUFnH2cWku_Th1hwt4TDdoMd8pWYVDzJeqJslMNZwpO3y0Et94d31qkNOuenGA$" target="_blank">https://www.cse.buffalo.edu/~knepley/</a><o:p></o:p></p>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
<div>
<p class="MsoNormal"><ksp_petsc_log.txt><o:p></o:p></p>
</div>
</div>
</blockquote>
</div>
<div>
<p class="MsoNormal"> <o:p></o:p></p>
</div>
</div>
</div>
</div>
</div>
<p class="MsoNormal"><ksp_petsc_log.txt><ksp_petsc_log_noguess.txt><o:p></o:p></p>
</div>
</blockquote>
</div>
<p class="MsoNormal"><o:p> </o:p></p>
</div>
</div>
</div>
</div>
</body>
</html>