File indexing completed on 2025-12-21 17:51:01 UTC
view on githubraw file Latest commit feb7fa5d on 2025-11-21 15:45:20 UTC
5fc0cd3689 Dani*0001 #include "STREAMICE_OPTIONS.h"
0002
0003
0004
0005
07e785229e dngo*0006 SUBROUTINE STREAMICE_CG_SOLVE_PETSC(
5fc0cd3689 Dani*0007 U cg_Uin,
0008 U cg_Vin,
0009 I cg_Bu,
0010 I cg_Bv,
0011 I A_uu,
0012 I A_uv,
0013 I A_vu,
0014 I A_vv,
07e785229e dngo*0015 I tolerance,
5fc0cd3689 Dani*0016 O iters,
d2cdb9260d Dani*0017 I maxIter,
5fc0cd3689 Dani*0018 I myThid )
8527100523 Dani*0019
8a34959769 dngo*0020
07e785229e dngo*0021
5fc0cd3689 Dani*0022
8a34959769 dngo*0023
0024
0025 #ifdef ALLOW_PETSC
0026 #ifdef STREAMICE_PETSC_3_8
f0ff6e912a dngo*0027
0028
0029
0030
0031
0032
0033
0034
0035
0036
8a34959769 dngo*0037 #include "petsc/finclude/petsc.h"
0038 #include "petsc/finclude/petscvec.h"
0039 use petscvec
0040 #include "petsc/finclude/petscmat.h"
0041 use petscmat
0042 #include "petsc/finclude/petscksp.h"
0043 use petscksp
0044 #include "petsc/finclude/petscpc.h"
0045 use petscpc
f0ff6e912a dngo*0046 #include "STREAMICE_PETSC_MOD.h"
8a34959769 dngo*0047 IMPLICIT NONE
0048 # else
5fc0cd3689 Dani*0049 IMPLICIT NONE
8a34959769 dngo*0050 #include "finclude/petsc.h"
f0ff6e912a dngo*0051 #include "STREAMICE_PETSC_MOD.h"
8a34959769 dngo*0052 #endif
0053 #endif
5fc0cd3689 Dani*0054
0055 #include "SIZE.h"
0056 #include "EEPARAMS.h"
0057 #include "PARAMS.h"
0058 #include "STREAMICE.h"
feb7fa5d1e dngo*0059 #include "STREAMICE_FP.h"
5fc0cd3689 Dani*0060 #include "STREAMICE_CG.h"
0061
0062
0063
0064
0065
0066
0067 INTEGER myThid
d2cdb9260d Dani*0068 INTEGER iters, maxiter
5fc0cd3689 Dani*0069 _RL tolerance
0070 _RL cg_Uin (1-OLx:sNx+OLx,1-OLy:sNy+OLy,nSx,nSy)
0071 _RL cg_Vin (1-OLx:sNx+OLx,1-OLy:sNy+OLy,nSx,nSy)
0072 _RL cg_Bu (1-OLx:sNx+OLx,1-OLy:sNy+OLy,nSx,nSy)
0073 _RL cg_Bv (1-OLx:sNx+OLx,1-OLy:sNy+OLy,nSx,nSy)
07e785229e dngo*0074 _RL
5fc0cd3689 Dani*0075 & A_uu (1-OLx:sNx+OLx,1-OLy:sNy+OLy,nSx,nSy,-1:1,-1:1),
0076 & A_vu (1-OLx:sNx+OLx,1-OLy:sNy+OLy,nSx,nSy,-1:1,-1:1),
0077 & A_uv (1-OLx:sNx+OLx,1-OLy:sNy+OLy,nSx,nSy,-1:1,-1:1),
0078 & A_vv (1-OLx:sNx+OLx,1-OLy:sNy+OLy,nSx,nSy,-1:1,-1:1)
0079
feb7fa5d1e dngo*0080 #ifdef ALLOW_AUTODIFF
0081 #ifdef ALLOW_STREAMICE_FP_ADJ
8527100523 Dani*0082 LOGICAL create_mat, destroy_mat
0083 #endif
feb7fa5d1e dngo*0084 #endif
8527100523 Dani*0085
07e785229e dngo*0086 #ifdef ALLOW_STREAMICE
0087 #ifdef ALLOW_PETSC
5fc0cd3689 Dani*0088
0089 INTEGER i, j, bi, bj, cg_halo, conv_flag
0090 INTEGER iter, is, js, ie, je, colx, coly, k
07e785229e dngo*0091 _RL dot_p1, dot_p2, resid, resid_0
5fc0cd3689 Dani*0092 _RL dot_p1_tile (nSx,nSy)
0093 _RL dot_p2_tile (nSx,nSy)
0094 CHARACTER*(MAX_LEN_MBUF) msgBuf
0095
07e785229e dngo*0096 INTEGER indices(2*(sNx*nSx*sNy*nSy))
5fc0cd3689 Dani*0097 INTEGER n_dofs_cum_sum (0:nPx*nPy-1), idx(1)
07e785229e dngo*0098 _RL rhs_values(2*(sNx*nSx*sNy*nSy))
0099 _RL solution_values(2*(sNx*nSx*sNy*nSy))
0100
5fc0cd3689 Dani*0101 _RL mat_values (18,1), mat_val_return(1)
0102 INTEGER indices_col(18)
0103 INTEGER local_dofs, global_dofs, dof_index, dof_index_col
0104 INTEGER local_offset
8a34959769 dngo*0105 #ifdef STREAMICE_PETSC_3_8
0106 INTEGER local_null
0107 #endif
07e785229e dngo*0108
0109
0110
08f6c3ab35 Dani*0111 PC subpc
8a34959769 dngo*0112 #ifdef STREAMICE_PETSC_3_8
0113 KSP subksp
0114 #else
8527100523 Dani*0115 KSP subksp(1)
8a34959769 dngo*0116 #endif
5fc0cd3689 Dani*0117 Vec rhs
0118 Vec solution
0119 PetscErrorCode ierr
0120 #ifdef ALLOW_USE_MPI
0121 integer mpiRC, mpiMyWid
0122 #endif
0123
0124 #ifdef ALLOW_USE_MPI
0125
0126 CALL MPI_COMM_RANK( MPI_COMM_WORLD, mpiMyWId, mpiRC )
0127 local_dofs = n_dofs_process (mpiMyWid)
0128 global_dofs = 0
07e785229e dngo*0129
5fc0cd3689 Dani*0130 n_dofs_cum_sum(0) = 0
0131 DO i=0,nPx*nPy-1
0132 global_dofs = global_dofs + n_dofs_process (i)
0133 if (i.ge.1) THEN
0134 n_dofs_cum_sum(i) = n_dofs_cum_sum(i-1)+
0135 & n_dofs_process(i-1)
0136 endif
0137 ENDDO
0138 local_offset = n_dofs_cum_sum(mpimywid)
0139
0140 #else
0141
0142 local_dofs = n_dofs_process (0)
0143 global_dofs = local_dofs
0144 local_offset = 0
0145
07e785229e dngo*0146 #endif
0147
5fc0cd3689 Dani*0148
07e785229e dngo*0149
5fc0cd3689 Dani*0150
18a089944d Dani*0151 CALL TIMER_START ('STREAMICE_PETSC_SETUP',myThid)
0152
5fc0cd3689 Dani*0153 call VecCreate(PETSC_COMM_WORLD, rhs, ierr)
0154 call VecSetSizes(rhs, local_dofs, global_dofs, ierr)
0155 call VecSetType(rhs, VECMPI, ierr)
0156
0157 call VecCreate(PETSC_COMM_WORLD, solution, ierr)
07e785229e dngo*0158 call VecSetSizes(solution, local_dofs, global_dofs, ierr)
5fc0cd3689 Dani*0159 call VecSetType(solution, VECMPI, ierr)
0160
0161 do i=1,local_dofs
0162 indices(i) = i-1 + local_offset
0163 end do
0164 do i=1,2*nSx*nSy*sNx*sNy
0165 rhs_values (i) = 0. _d 0
0166 solution_values (i) = 0. _d 0
0167 enddo
0168
07e785229e dngo*0169
5fc0cd3689 Dani*0170
0171 DO bj = myByLo(myThid), myByHi(myThid)
0172 DO bi = myBxLo(myThid), myBxHi(myThid)
0173 DO j=1,sNy
0174 DO i=1,sNx
0175
0176 dof_index = INT(streamice_petsc_dofs_u(i,j,bi,bj))
0177 & - local_offset
0178
0179 if (dof_index.ge.0) THEN
07e785229e dngo*0180
5fc0cd3689 Dani*0181 rhs_values(dof_index+1) = cg_Bu(i,j,bi,bj)
0182 solution_values(dof_index+1) = cg_Uin(i,j,bi,bj)
0183
0184 endif
0185
07e785229e dngo*0186
5fc0cd3689 Dani*0187
0188 dof_index = INT(streamice_petsc_dofs_v(i,j,bi,bj))
0189 & - local_offset
0190
0191 if (dof_index.ge.0) THEN
0192
0193 rhs_values(dof_index+1) = cg_Bv(i,j,bi,bj)
0194 solution_values(dof_index+1) = cg_Vin(i,j,bi,bj)
0195
0196 endif
0197
0198 ENDDO
0199 ENDDO
0200 ENDDO
0201 ENDDO
0202
0203 call VecSetValues(rhs, local_dofs, indices, rhs_values,
0204 & INSERT_VALUES, ierr)
0205 call VecAssemblyBegin(rhs, ierr)
0206 call VecAssemblyEnd(rhs, ierr)
0207
0208 call VecSetValues(solution, local_dofs, indices,
0209 & solution_values, INSERT_VALUES, ierr)
0210 call VecAssemblyBegin(solution, ierr)
0211 call VecAssemblyEnd(solution, ierr)
0212
feb7fa5d1e dngo*0213 #ifdef ALLOW_AUTODIFF
0214 #ifdef ALLOW_STREAMICE_FP_ADJ
8527100523 Dani*0215 #ifdef ALLOW_PETSC
0216 if (STREAMICE_need2createmat) then
0217 #endif
0218 #endif
feb7fa5d1e dngo*0219 #endif
8527100523 Dani*0220
07e785229e dngo*0221
0222
0223 call MatCreateAIJ (PETSC_COMM_WORLD,
5fc0cd3689 Dani*0224 & local_dofs, local_dofs,
07e785229e dngo*0225 & global_dofs, global_dofs,
0226 & 18, PETSC_NULL_INTEGER,
5fc0cd3689 Dani*0227 & 18, PETSC_NULL_INTEGER,
0228 & matrix, ierr)
0229
07e785229e dngo*0230
5fc0cd3689 Dani*0231
0232 DO bj = myByLo(myThid), myByHi(myThid)
0233 DO bi = myBxLo(myThid), myBxHi(myThid)
0234 DO j=1,sNy
0235 DO i=1,sNx
0236
0237 dof_index = INT(streamice_petsc_dofs_u(i,j,bi,bj))
07e785229e dngo*0238
5fc0cd3689 Dani*0239
0240 IF (dof_index .ge. 0) THEN
0241
0242 DO k=1,18
0243 indices_col(k) = 0
0244 mat_values(k,1) = 0. _d 0
0245 ENDDO
0246 k=0
0247
0248 DO coly=-1,1
0249 DO colx=-1,1
0250
0251 dof_index_col = streamice_petsc_dofs_u(i+colx,j+coly,bi,bj)
0252
0253 if (dof_index_col.ge.0) THEN
07e785229e dngo*0254
0255
0256
5fc0cd3689 Dani*0257 k=k+1
0258 mat_values (k,1) = A_uu(i,j,bi,bj,colx,coly)
0259 indices_col (k) = dof_index_col
0260 endif
07e785229e dngo*0261
5fc0cd3689 Dani*0262 dof_index_col = streamice_petsc_dofs_v(i+colx,j+coly,bi,bj)
0263
0264 if (dof_index_col.ge.0) THEN
07e785229e dngo*0265
0266
5fc0cd3689 Dani*0267 k=k+1
0268 mat_values (k,1) = A_uv(i,j,bi,bj,colx,coly)
0269 indices_col (k) = dof_index_col
0270 endif
07e785229e dngo*0271
5fc0cd3689 Dani*0272 ENDDO
0273 ENDDO
0274
8a34959769 dngo*0275 #ifdef STREAMICE_PETSC_3_8
0276 call MatSetValues1n (matrix, 1, dof_index, k, indices_col,
0277 #else
0278 call MatSetValues (matrix, 1, dof_index, k, indices_col,
0279 #endif
5fc0cd3689 Dani*0280 & mat_values,INSERT_VALUES,ierr)
0281
0282 ENDIF
0283
07e785229e dngo*0284
5fc0cd3689 Dani*0285
0286 dof_index = INT(streamice_petsc_dofs_v(i,j,bi,bj))
07e785229e dngo*0287
5fc0cd3689 Dani*0288
0289 IF (dof_index .ge. 0) THEN
0290
0291 DO k=1,18
0292 indices_col(k) = 0
0293 mat_values(k,1) = 0. _d 0
0294 ENDDO
0295 k=0
0296
0297 DO coly=-1,1
0298 DO colx=-1,1
0299
0300 dof_index_col = streamice_petsc_dofs_u(i+colx,j+coly,bi,bj)
0301
0302 if (dof_index_col.ge.0) THEN
07e785229e dngo*0303
0304
5fc0cd3689 Dani*0305 k=k+1
0306 mat_values (k,1) = A_vu(i,j,bi,bj,colx,coly)
0307 indices_col (k) = dof_index_col
0308 endif
0309
0310 dof_index_col = streamice_petsc_dofs_v(i+colx,j+coly,bi,bj)
0311
0312 if (dof_index_col.ge.0) THEN
07e785229e dngo*0313
0314
5fc0cd3689 Dani*0315 k=k+1
0316 mat_values (k,1) = A_vv(i,j,bi,bj,colx,coly)
0317 indices_col (k) = dof_index_col
0318 endif
0319
0320 ENDDO
0321 ENDDO
0322
8a34959769 dngo*0323 #ifdef STREAMICE_PETSC_3_8
0324 call MatSetValues1n (matrix, 1, dof_index, k, indices_col,
0325 #else
0326 call MatSetValues (matrix, 1, dof_index, k, indices_col,
0327 #endif
5fc0cd3689 Dani*0328 & mat_values,INSERT_VALUES,ierr)
0329 ENDIF
0330
0331 ENDDO
0332 ENDDO
0333 ENDDO
0334 ENDDO
0335
0336 call MatAssemblyBegin(matrix,MAT_FINAL_ASSEMBLY,ierr)
0337 call MatAssemblyEnd(matrix,MAT_FINAL_ASSEMBLY,ierr)
0338
0339 call KSPCreate(PETSC_COMM_WORLD, ksp, ierr)
8a34959769 dngo*0340 #ifdef STREAMICE_PETSC_3_8
07e785229e dngo*0341 call KSPSetOperators(ksp, matrix, matrix,
8a34959769 dngo*0342 & ierr)
0343 #else
0344 call KSPSetOperators(ksp, matrix, matrix,
0345 & DIFFERENT_NONZERO_PATTERN,ierr)
0346 #endif
5fc0cd3689 Dani*0347
8527100523 Dani*0348 IF (PETSC_PRECOND_TYPE.eq.'MUMPS') then
0349 call KSPSetType(ksp,KSPPREONLY,ierr)
0350 ELSE
0351 SELECT CASE (PETSC_SOLVER_TYPE)
5fc0cd3689 Dani*0352 CASE ('CG')
0353 PRINT *, "PETSC SOLVER: SELECTED CG"
0354 call KSPSetType(ksp, KSPCG, ierr)
0355 CASE ('GMRES')
0356 PRINT *, "PETSC SOLVER: SELECTED GMRES"
0357 call KSPSetType(ksp, KSPGMRES, ierr)
0358 CASE ('BICG')
0359 PRINT *, "PETSC SOLVER: SELECTED BICG"
07e785229e dngo*0360 call KSPSetType(ksp, KSPBCGS, ierr)
5fc0cd3689 Dani*0361 CASE DEFAULT
0362 PRINT *, "PETSC SOLVER: SELECTED DEFAULT"
0363 call KSPSetType(ksp, KSPCG, ierr)
8527100523 Dani*0364 END SELECT
0365 ENDIF
5fc0cd3689 Dani*0366
0367 call KSPGetPC(ksp, pc, ierr)
0368 call KSPSetTolerances(ksp,tolerance,
07e785229e dngo*0369 & PETSC_DEFAULT_REAL,
0370 & PETSC_DEFAULT_REAL,
d2cdb9260d Dani*0371 & maxiter,ierr)
5fc0cd3689 Dani*0372
0373 SELECT CASE (PETSC_PRECOND_TYPE)
0374 CASE ('BLOCKJACOBI')
0375 PRINT *, "PETSC PRECOND: SELECTED BJACOBI"
0376 call PCSetType(pc, PCBJACOBI, ierr)
08f6c3ab35 Dani*0377 call kspsetup (ksp, ierr)
8a34959769 dngo*0378 #ifdef STREAMICE_PETSC_3_8
0379 local_null =0
0380 call PCBJacobiGetSubKSP1 (pc,local_null,local_null,
0381 #else
0382 call PCBJacobiGetSubKSP1 (pc,PETSC_NULL_INTEGER,
0383 & PETSC_NULL_INTEGER,
0384 #endif
08f6c3ab35 Dani*0385 & subksp,ierr);
0386 call KSPGetPC (subksp, subpc, ierr)
07e785229e dngo*0387 call PCSetType (subpc, PCICC, ierr)
0388 call PCFactorSetLevels(subpc,streamice_petsc_pcfactorlevels,
0389 & ierr)
5fc0cd3689 Dani*0390 CASE ('JACOBI')
0391 PRINT *, "PETSC PRECOND: SELECTED JACOBI"
0392 call PCSetType(pc, PCJACOBI, ierr)
0393 CASE ('ILU')
0394 PRINT *, "PETSC PRECOND: SELECTED ILU"
0395 call PCSetType(pc, PCILU, ierr)
18a089944d Dani*0396
0397 CASE ('GAMG')
0398 PRINT *, "PETSC PRECOND: SELECTED GAMG"
0399 call PCSetType(pc, PCGAMG, ierr)
0400 call PCGAMGSetCoarseEqLim(pc,10000,ierr)
96b006450c dngo*0401
18a089944d Dani*0402 call PCGAMGSetNSmooths(pc, 0,ierr)
8a34959769 dngo*0403
0404
8527100523 Dani*0405 call kspsetup (ksp, ierr)
0406
0407 CASE ('MUMPS')
0408 PRINT *, "PETSC PRECOND: SELECTED MUMPS"
0409 call PCSetType(pc,PCLU,ierr)
8a34959769 dngo*0410 call PCFactorSetMatSolverType(pc,MATSOLVERMUMPS,ierr)
0411 call PCFactorSetUpMatSolverType(pc,ierr)
8527100523 Dani*0412 call PCFactorGetMatrix(pc,mumpsFac,ierr)
0413 call MatMumpsSetIcntl(mumpsfac,24,1,ierr)
0414 call kspsetup (ksp, ierr)
18a089944d Dani*0415
5fc0cd3689 Dani*0416 CASE DEFAULT
0417 PRINT *, "PETSC PRECOND: SELECTED DEFAULT"
0418 call PCSetType(pc, PCBJACOBI, ierr)
0419 END SELECT
0420
18a089944d Dani*0421 CALL TIMER_STOP ('STREAMICE_PETSC_SETUP',myThid)
feb7fa5d1e dngo*0422 #ifdef ALLOW_AUTODIFF
0423 #ifdef ALLOW_STREAMICE_FP_ADJ
8527100523 Dani*0424 #ifdef ALLOW_PETSC
0425 endif
0426 #endif
feb7fa5d1e dngo*0427 #endif
8527100523 Dani*0428 #endif
0429
18a089944d Dani*0430 CALL TIMER_START ('STREAMICE_PETSC_SOLVE',myThid)
0431
5fc0cd3689 Dani*0432 call KSPSolve(ksp, rhs, solution, ierr)
18a089944d Dani*0433
0434 CALL TIMER_STOP ('STREAMICE_PETSC_SOLVE',myThid)
0435
5fc0cd3689 Dani*0436 call KSPGetIterationNumber(ksp,iters,ierr)
0437
0438 call VecGetValues(solution,local_dofs,indices,
0439 & solution_values,ierr)
0440
0441 DO bj = myByLo(myThid), myByHi(myThid)
0442 DO bi = myBxLo(myThid), myBxHi(myThid)
0443 DO j=1,sNy
0444 DO i=1,sNx
0445
0446 dof_index = INT(streamice_petsc_dofs_u(i,j,bi,bj))
0447 & - local_offset
0448 if (dof_index.ge.0) THEN
0449 cg_Uin(i,j,bi,bj) = solution_values(dof_index+1)
0450 endif
07e785229e dngo*0451
5fc0cd3689 Dani*0452 dof_index = INT(streamice_petsc_dofs_v(i,j,bi,bj))
0453 & - local_offset
0454 if (dof_index.ge.0) THEN
0455 cg_Vin(i,j,bi,bj) = solution_values(dof_index+1)
0456 endif
0457
0458 ENDDO
0459 ENDDO
0460 ENDDO
0461 ENDDO
0462
feb7fa5d1e dngo*0463 #ifdef ALLOW_AUTODIFF
0464 #ifdef ALLOW_STREAMICE_FP_ADJ
8527100523 Dani*0465 if (streamice_need2destroymat) then
feb7fa5d1e dngo*0466 #endif
8527100523 Dani*0467 #endif
5fc0cd3689 Dani*0468 call KSPDestroy (ksp, ierr)
8527100523 Dani*0469 call MatDestroy (matrix, ierr)
feb7fa5d1e dngo*0470 #ifdef ALLOW_AUTODIFF
0471 #ifdef ALLOW_STREAMICE_FP_ADJ
8527100523 Dani*0472 endif
feb7fa5d1e dngo*0473 #endif
8527100523 Dani*0474 #endif
5fc0cd3689 Dani*0475 call VecDestroy (rhs, ierr)
0476 call VecDestroy (solution, ierr)
0477
07e785229e dngo*0478
5fc0cd3689 Dani*0479
07e785229e dngo*0480 #endif /* ALLOW_PETSC */
0481 #endif /* ALLOW_STREAMICE */
5fc0cd3689 Dani*0482 RETURN
0483 END